github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-17 16:11:26 -05:00
Code Issues Packages Projects Releases Wiki Activity

test(compiler): Add retry for fixing some flaky tests

Browse Source
This commit is contained in:
Bourgerie Quentin
2023-11-13 16:06:49 +01:00
committed by rudy-6-4
parent 835d062e12
commit 588075ddc8
4 changed files with 63 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
compilers/concrete-compiler/compiler/Makefile
View File

@@ -299,7 +299,7 @@ run-end-to-end-tests: $(GTEST_PARALLEL_PY) build-end-to-end-tests generate-cpu-t
$(foreach optimizer_strategy,$(OPTIMIZATION_STRATEGY_TO_TEST), $(foreach security,$(SECURITY_TO_TEST), \
$(GTEST_PARALLEL_CMD) $(BUILD_DIR)/tools/concretelang/tests/end_to_end_tests/end_to_end_test \
$(GTEST_PARALLEL_SEPARATOR) --backend=cpu --security-level=$(security) \
--optimizer-strategy=$(optimizer_strategy) $(FIXTURE_CPU_DIR)/*.yaml || exit $$?;))
--optimizer-strategy=$(optimizer_strategy) --retry-failing-tests=1 $(FIXTURE_CPU_DIR)/*.yaml || exit $$?;))
### end-to-end-tests GPU

2
compilers/concrete-compiler/compiler/include/concretelang/TestLib/TestCircuit.h
View File

@@ -91,6 +91,8 @@ public:
std::string getArtifactFolder() { return artifactFolder; }
void setKeySet(Keyset keyset) { this->keyset = keyset; }
private:
TestCircuit(ClientCircuit clientCircuit, ServerCircuit serverCircuit,
bool useSimulation, std::string artifactFolder, Keyset keyset)

77
compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.cc
View File

@@ -25,9 +25,11 @@ class EndToEndTest : public ::testing::Test {
public:
explicit EndToEndTest(std::string program, TestDescription desc,
std::optional<TestErrorRate> errorRate,
mlir::concretelang::CompilationOptions options)
mlir::concretelang::CompilationOptions options,
int retryFailingTests)
: program(program), desc(desc), errorRate(errorRate),
testCircuit(std::nullopt), options(options) {
testCircuit(std::nullopt), options(options),
retryFailingTests(retryFailingTests) {
if (errorRate.has_value()) {
options.optimizerConfig.global_p_error = errorRate->global_p_error;
options.optimizerConfig.p_error = errorRate->global_p_error;
@@ -43,19 +45,19 @@ public:
ce.setCompilationOptions(options);
auto expectCompilationResult = ce.compile({program}, artifactFolder);
ASSERT_EXPECTED_SUCCESS(expectCompilationResult);
auto compiled = expectCompilationResult.get();
library = expectCompilationResult.get();
/* Retrieve the keyset */
auto keyset =
getTestKeySetCachePtr()
->getKeyset(compiled.getProgramInfo().asReader().getKeyset(), 0, 0)
->getKeyset(library->getProgramInfo().asReader().getKeyset(), 0, 0)
.value();
/* Create the test circuit */
testCircuit =
TestCircuit::create(
keyset, compiled.getProgramInfo().asReader(),
compiled.getSharedLibraryPath(compiled.getOutputDirPath()), 0, 0,
keyset, library->getProgramInfo().asReader(),
library->getSharedLibraryPath(library->getOutputDirPath()), 0, 0,
false)
.value();
@@ -77,14 +79,31 @@ public:
}
void testOnce() {
// We execute the circuit.
auto maybeRes = (*testCircuit).call(args);
ASSERT_OUTCOME_HAS_VALUE(maybeRes);
auto result = maybeRes.value();
for (auto tests_rep = 0; tests_rep <= retryFailingTests; tests_rep++) {
// We execute the circuit.
auto maybeRes = (*testCircuit).call(args);
ASSERT_OUTCOME_HAS_VALUE(maybeRes);
auto result = maybeRes.value();
/* Check result */
for (size_t i = 0; i < desc.outputs.size(); i++) {
ASSERT_LLVM_ERROR(checkResult(desc.outputs[i], result[i]));
/* Check result */
for (size_t i = 0; i < desc.outputs.size(); i++) {
auto maybeErr = checkResult(desc.outputs[i], result[i]);
if (!maybeErr)
return;
if (tests_rep < retryFailingTests) {
llvm::errs() << "/!\\ WARNING RETRY TEST: " << maybeErr << "\n";
llvm::consumeError(std::move(maybeErr));
llvm::errs() << "Regenerating keyset\n";
__uint128_t seed = tests_rep + 1;
auto csprng = ConcreteCSPRNG(seed);
Keyset keyset =
Keyset(library->getProgramInfo().asReader().getKeyset(), csprng);
testCircuit->setKeySet(keyset);
break;
} else {
ASSERT_LLVM_ERROR(std::move(maybeErr));
}
}
}
}
@@ -117,8 +136,10 @@ private:
std::string artifactFolder;
TestDescription desc;
std::optional<TestErrorRate> errorRate;
std::optional<mlir::concretelang::CompilerEngine::Library> library;
std::optional<TestCircuit> testCircuit;
mlir::concretelang::CompilationOptions options;
int retryFailingTests;
std::vector<Value> args;
};
@@ -134,19 +155,22 @@ void registerEndToEnd(std::string suiteName, std::string testName,
std::string valueName, std::string program,
TestDescription test,
std::optional<TestErrorRate> errorRate,
mlir::concretelang::CompilationOptions options) {
mlir::concretelang::CompilationOptions options,
int retryFailingTests) {
// TODO: Get file and line from yaml
auto file = __FILE__;
auto line = __LINE__;
::testing::RegisterTest(
suiteName.c_str(), testName.c_str(), nullptr, valueName.c_str(), file,
line, [=]() -> EndToEndTest * {
return new EndToEndTest(program, test, errorRate, options);
});
::testing::RegisterTest(suiteName.c_str(), testName.c_str(), nullptr,
valueName.c_str(), file, line,
[=]() -> EndToEndTest * {
return new EndToEndTest(program, test, errorRate,
options, retryFailingTests);
});
}
void registerEndToEnd(std::string suiteName, EndToEndDesc desc,
mlir::concretelang::CompilationOptions options) {
mlir::concretelang::CompilationOptions options,
int retryFailingTests) {
if (desc.v0Constraint.has_value()) {
options.v0FHEConstraints = desc.v0Constraint;
}
@@ -161,13 +185,13 @@ void registerEndToEnd(std::string suiteName, EndToEndDesc desc,
auto testName = getTestName(desc, options, i);
if (desc.test_error_rates.empty()) {
registerEndToEnd(suiteName, testName, valueName, desc.program, test,
std::nullopt, options);
std::nullopt, options, retryFailingTests);
} else {
auto j = 0;
for (auto rate : desc.test_error_rates) {
auto rateName = testName + "_rate" + std::to_string(j);
registerEndToEnd(suiteName, rateName, valueName, desc.program, test,
rate, options);
rate, options, retryFailingTests);
j++;
}
}
@@ -181,9 +205,10 @@ void registerEndToEnd(std::string suiteName, EndToEndDesc desc,
/// @param options The compilation options.
void registerEndToEndSuite(std::string suiteName,
std::vector<EndToEndDesc> descriptions,
mlir::concretelang::CompilationOptions options) {
mlir::concretelang::CompilationOptions options,
int retryFailingTests) {
for (auto desc : descriptions) {
registerEndToEnd(suiteName, desc, options);
registerEndToEnd(suiteName, desc, options, retryFailingTests);
}
}
@@ -200,10 +225,12 @@ int main(int argc, char **argv) {
auto compilationOptions = std::get<0>(options);
auto descriptionFiles = std::get<1>(options);
auto retryFailingTests = std::get<2>(options);
for (auto descFile : descriptionFiles) {
auto suiteName = path::stem(descFile.path).str() + ".library";
registerEndToEndSuite(suiteName, descFile.descriptions, compilationOptions);
registerEndToEndSuite(suiteName, descFile.descriptions, compilationOptions,
retryFailingTests);
}
return RUN_ALL_TESTS();
}

10
compilers/concrete-compiler/compiler/tests/end_to_end_tests/end_to_end_test.h
View File

@@ -23,7 +23,7 @@ const double TEST_ERROR_RATE = 1.0 - 0.999936657516;
/// options, the library path if the --library options has been specified and
/// the parsed description files
std::tuple<mlir::concretelang::CompilationOptions,
std::vector<EndToEndDescFile>>
std::vector<EndToEndDescFile>, int>
parseEndToEndCommandLine(int argc, char **argv) {
namespace optimizer = mlir::concretelang::optimizer;
// TODO - Well reset other llvm command line options registered but assert on
@@ -101,6 +101,11 @@ parseEndToEndCommandLine(int argc, char **argv) {
llvm::cl::desc("Set the compiler verbosity"),
llvm::cl::init(false));
// e2e test options
llvm::cl::opt<int> retryFailingTests("retry-failing-tests",
llvm::cl::desc("Retry test which fails"),
llvm::cl::init(0));
llvm::cl::ParseCommandLineOptions(argc, argv);
// Build compilation options
@@ -129,7 +134,8 @@ parseEndToEndCommandLine(int argc, char **argv) {
parsedDescriptionFiles.push_back(f);
}
return std::make_tuple(compilationOptions, parsedDescriptionFiles);
return std::make_tuple(compilationOptions, parsedDescriptionFiles,
retryFailingTests.getValue());
}
std::string getOptionsName(mlir::concretelang::CompilationOptions options) {