diff --git a/compilers/concrete-compiler/compiler/Makefile b/compilers/concrete-compiler/compiler/Makefile index 53603f61c..ca80ac869 100644 --- a/compilers/concrete-compiler/compiler/Makefile +++ b/compilers/concrete-compiler/compiler/Makefile @@ -299,7 +299,7 @@ run-end-to-end-tests: $(GTEST_PARALLEL_PY) build-end-to-end-tests generate-cpu-t $(foreach optimizer_strategy,$(OPTIMIZATION_STRATEGY_TO_TEST), $(foreach security,$(SECURITY_TO_TEST), \ $(GTEST_PARALLEL_CMD) $(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_test \ $(GTEST_PARALLEL_SEPARATOR) --backend=cpu --security-level=$(security) \ - --optimizer-strategy=$(optimizer_strategy) $(FIXTURE_CPU_DIR)/*.yaml || exit $$?;)) + --optimizer-strategy=$(optimizer_strategy) --retry-failing-tests=1 $(FIXTURE_CPU_DIR)/*.yaml || exit $$?;)) ### end-to-end-tests GPU diff --git a/compilers/concrete-compiler/compiler/include/concretelang/TestLib/TestCircuit.h b/compilers/concrete-compiler/compiler/include/concretelang/TestLib/TestCircuit.h index 6efd1d29d..48996252c 100644 --- a/compilers/concrete-compiler/compiler/include/concretelang/TestLib/TestCircuit.h +++ b/compilers/concrete-compiler/compiler/include/concretelang/TestLib/TestCircuit.h @@ -91,6 +91,8 @@ public: std::string getArtifactFolder() { return artifactFolder; } + void setKeySet(Keyset keyset) { this->keyset = keyset; } + private: TestCircuit(ClientCircuit clientCircuit, ServerCircuit serverCircuit, bool useSimulation, std::string artifactFolder, Keyset keyset) diff --git a/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.cc b/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.cc index d2930d266..40caeefbc 100644 --- a/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.cc +++ b/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.cc @@ -25,9 +25,11 @@ class EndToEndTest : public ::testing::Test { public: explicit EndToEndTest(std::string program, TestDescription desc, std::optional errorRate, - mlir::concretelang::CompilationOptions options) + mlir::concretelang::CompilationOptions options, + int retryFailingTests) : program(program), desc(desc), errorRate(errorRate), - testCircuit(std::nullopt), options(options) { + testCircuit(std::nullopt), options(options), + retryFailingTests(retryFailingTests) { if (errorRate.has_value()) { options.optimizerConfig.global_p_error = errorRate->global_p_error; options.optimizerConfig.p_error = errorRate->global_p_error; @@ -43,19 +45,19 @@ public: ce.setCompilationOptions(options); auto expectCompilationResult = ce.compile({program}, artifactFolder); ASSERT_EXPECTED_SUCCESS(expectCompilationResult); - auto compiled = expectCompilationResult.get(); + library = expectCompilationResult.get(); /* Retrieve the keyset */ auto keyset = getTestKeySetCachePtr() - ->getKeyset(compiled.getProgramInfo().asReader().getKeyset(), 0, 0) + ->getKeyset(library->getProgramInfo().asReader().getKeyset(), 0, 0) .value(); /* Create the test circuit */ testCircuit = TestCircuit::create( - keyset, compiled.getProgramInfo().asReader(), - compiled.getSharedLibraryPath(compiled.getOutputDirPath()), 0, 0, + keyset, library->getProgramInfo().asReader(), + library->getSharedLibraryPath(library->getOutputDirPath()), 0, 0, false) .value(); @@ -77,14 +79,31 @@ public: } void testOnce() { - // We execute the circuit. - auto maybeRes = (*testCircuit).call(args); - ASSERT_OUTCOME_HAS_VALUE(maybeRes); - auto result = maybeRes.value(); + for (auto tests_rep = 0; tests_rep <= retryFailingTests; tests_rep++) { + // We execute the circuit. + auto maybeRes = (*testCircuit).call(args); + ASSERT_OUTCOME_HAS_VALUE(maybeRes); + auto result = maybeRes.value(); - /* Check result */ - for (size_t i = 0; i < desc.outputs.size(); i++) { - ASSERT_LLVM_ERROR(checkResult(desc.outputs[i], result[i])); + /* Check result */ + for (size_t i = 0; i < desc.outputs.size(); i++) { + auto maybeErr = checkResult(desc.outputs[i], result[i]); + if (!maybeErr) + return; + if (tests_rep < retryFailingTests) { + llvm::errs() << "/!\\ WARNING RETRY TEST: " << maybeErr << "\n"; + llvm::consumeError(std::move(maybeErr)); + llvm::errs() << "Regenerating keyset\n"; + __uint128_t seed = tests_rep + 1; + auto csprng = ConcreteCSPRNG(seed); + Keyset keyset = + Keyset(library->getProgramInfo().asReader().getKeyset(), csprng); + testCircuit->setKeySet(keyset); + break; + } else { + ASSERT_LLVM_ERROR(std::move(maybeErr)); + } + } } } @@ -117,8 +136,10 @@ private: std::string artifactFolder; TestDescription desc; std::optional errorRate; + std::optional library; std::optional testCircuit; mlir::concretelang::CompilationOptions options; + int retryFailingTests; std::vector args; }; @@ -134,19 +155,22 @@ void registerEndToEnd(std::string suiteName, std::string testName, std::string valueName, std::string program, TestDescription test, std::optional errorRate, - mlir::concretelang::CompilationOptions options) { + mlir::concretelang::CompilationOptions options, + int retryFailingTests) { // TODO: Get file and line from yaml auto file = __FILE__; auto line = __LINE__; - ::testing::RegisterTest( - suiteName.c_str(), testName.c_str(), nullptr, valueName.c_str(), file, - line, [=]() -> EndToEndTest * { - return new EndToEndTest(program, test, errorRate, options); - }); + ::testing::RegisterTest(suiteName.c_str(), testName.c_str(), nullptr, + valueName.c_str(), file, line, + [=]() -> EndToEndTest * { + return new EndToEndTest(program, test, errorRate, + options, retryFailingTests); + }); } void registerEndToEnd(std::string suiteName, EndToEndDesc desc, - mlir::concretelang::CompilationOptions options) { + mlir::concretelang::CompilationOptions options, + int retryFailingTests) { if (desc.v0Constraint.has_value()) { options.v0FHEConstraints = desc.v0Constraint; } @@ -161,13 +185,13 @@ void registerEndToEnd(std::string suiteName, EndToEndDesc desc, auto testName = getTestName(desc, options, i); if (desc.test_error_rates.empty()) { registerEndToEnd(suiteName, testName, valueName, desc.program, test, - std::nullopt, options); + std::nullopt, options, retryFailingTests); } else { auto j = 0; for (auto rate : desc.test_error_rates) { auto rateName = testName + "_rate" + std::to_string(j); registerEndToEnd(suiteName, rateName, valueName, desc.program, test, - rate, options); + rate, options, retryFailingTests); j++; } } @@ -181,9 +205,10 @@ void registerEndToEnd(std::string suiteName, EndToEndDesc desc, /// @param options The compilation options. void registerEndToEndSuite(std::string suiteName, std::vector descriptions, - mlir::concretelang::CompilationOptions options) { + mlir::concretelang::CompilationOptions options, + int retryFailingTests) { for (auto desc : descriptions) { - registerEndToEnd(suiteName, desc, options); + registerEndToEnd(suiteName, desc, options, retryFailingTests); } } @@ -200,10 +225,12 @@ int main(int argc, char **argv) { auto compilationOptions = std::get<0>(options); auto descriptionFiles = std::get<1>(options); + auto retryFailingTests = std::get<2>(options); for (auto descFile : descriptionFiles) { auto suiteName = path::stem(descFile.path).str() + ".library"; - registerEndToEndSuite(suiteName, descFile.descriptions, compilationOptions); + registerEndToEndSuite(suiteName, descFile.descriptions, compilationOptions, + retryFailingTests); } return RUN_ALL_TESTS(); } diff --git a/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.h b/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.h index c075fe0df..0101169a9 100644 --- a/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.h +++ b/compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.h @@ -23,7 +23,7 @@ const double TEST_ERROR_RATE = 1.0 - 0.999936657516; /// options, the library path if the --library options has been specified and /// the parsed description files std::tuple> + std::vector, int> parseEndToEndCommandLine(int argc, char **argv) { namespace optimizer = mlir::concretelang::optimizer; // TODO - Well reset other llvm command line options registered but assert on @@ -101,6 +101,11 @@ parseEndToEndCommandLine(int argc, char **argv) { llvm::cl::desc("Set the compiler verbosity"), llvm::cl::init(false)); + // e2e test options + llvm::cl::opt retryFailingTests("retry-failing-tests", + llvm::cl::desc("Retry test which fails"), + llvm::cl::init(0)); + llvm::cl::ParseCommandLineOptions(argc, argv); // Build compilation options @@ -129,7 +134,8 @@ parseEndToEndCommandLine(int argc, char **argv) { parsedDescriptionFiles.push_back(f); } - return std::make_tuple(compilationOptions, parsedDescriptionFiles); + return std::make_tuple(compilationOptions, parsedDescriptionFiles, + retryFailingTests.getValue()); } std::string getOptionsName(mlir::concretelang::CompilationOptions options) {