diff --git a/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp b/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp index 024ef7f71..53164346a 100644 --- a/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp +++ b/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp @@ -1,3 +1,5 @@ +#include + #include "EndToEndFixture.h" #include "concretelang/Support/CompilerEngine.h" #include "concretelang/Support/Jit.h" @@ -6,6 +8,61 @@ using mlir::concretelang::StreamStringError; +// derived from https://stackoverflow.com/a/45869209 +uint64_t solve_binomial_cdf_bigger_than(size_t n, double p_error, + double p_mass) { + // Solve: find k such that + // - binomial_cdf(k, n, p_error) >= p_mass + // - given n and p_error and p_mass + // Notation: + // - n is the number of repetition, + // - p_error the probability of error, + // - p_mass is the mass probability of having a number of error <= k + // - k is a number of error that is returned + // This returns the smallest threshold for error such that the number of error + // is below or equal, with probability p_mass. + using std::exp; + using std::log; + auto log_p_error = log(p_error); + auto log_p_success = log(1.0 - p_error); + auto cdf_k = 0.0; + double log_pmf_k; + // k in 0..n, isum and stop when the cumulative attained p_mass + for (uint64_t k = 0; k < n; k++) { + if (k == 0) { + // start with n success + log_pmf_k = n * log_p_success; + } else { + // add one error and remove one success + log_pmf_k += log(n - k + 1) - log(k) + (log_p_error - log_p_success); + } + cdf_k += exp(log_pmf_k); + if (cdf_k > p_mass) { + return k; + } + } + return n; +} + +uint64_t TestErrorRate::too_high_error_count_threshold() { + // Return the smallest threshold for error such that + // the number of error is higher than this threshold + // with probability FALSE_ALARM_RATE. + + // Examples: + // n_rep=100, p_error=0.05 -> 16 error max (ratio 3) + // n_rep=100, p_error=0.01 -> 7 error max (ratio 7) + // n_rep=100, p_error=0.001 -> 3 error max (ratio 30) + // n_rep=10000, p_error=0.0001 -> 8 error max (ratio 8) + // A high ratio indicate that the number of repetition should be increased. + // A good ratio (but costly) is between 1 and 2. + // A bad ratio can still detect most issues. + // A good ratio will help to detect more precise calibration issue. + double p_mass = 1.0 - TestErrorRate::FALSE_ALARM_RATE; + return solve_binomial_cdf_bigger_than(this->nb_repetition, this->p_error, + p_mass); +} + llvm::Expected scalarDescToLambdaArgument(ScalarDesc desc) { switch (desc.width) { @@ -176,6 +233,15 @@ template <> struct llvm::yaml::MappingTraits { LLVM_YAML_IS_SEQUENCE_VECTOR(TestDescription) +template <> struct llvm::yaml::MappingTraits { + static void mapping(IO &io, TestErrorRate &desc) { + io.mapRequired("p-error", desc.p_error); + io.mapRequired("nb-repetition", desc.nb_repetition); + } +}; + +LLVM_YAML_IS_SEQUENCE_VECTOR(TestErrorRate) + template <> struct llvm::yaml::MappingTraits { static void mapping(IO &io, EndToEndDesc &desc) { io.mapRequired("description", desc.description); @@ -210,6 +276,7 @@ template <> struct llvm::yaml::MappingTraits { if (!largeInterger.crtDecomposition.empty()) { desc.largeIntegerParameter = largeInterger; } + io.mapOptional("test-error-rates", desc.test_error_rates); } }; @@ -227,7 +294,9 @@ std::vector loadEndToEndDesc(std::string path) { yin >> desc; // Check for error - if (yin.error()) - assert(false && "cannot parse doc"); + if (yin.error()) { // .error() displays the beginning of the error message + std::cerr << "In yaml file: " << path << "\n"; + assert(false); + } return desc; } diff --git a/compiler/tests/end_to_end_fixture/EndToEndFixture.h b/compiler/tests/end_to_end_fixture/EndToEndFixture.h index b458710eb..d4f989701 100644 --- a/compiler/tests/end_to_end_fixture/EndToEndFixture.h +++ b/compiler/tests/end_to_end_fixture/EndToEndFixture.h @@ -37,6 +37,14 @@ struct TestDescription { std::vector outputs; }; +struct TestErrorRate { + double p_error; + uint64_t nb_repetition; + // rate at which a valid code will make the test fail due to bad luck + const double FALSE_ALARM_RATE = 0.00001; + uint64_t too_high_error_count_threshold(); +}; + struct EndToEndDesc { std::string description; std::string program; @@ -45,6 +53,7 @@ struct EndToEndDesc { llvm::Optional v0Constraint; llvm::Optional largeIntegerParameter; + std::vector test_error_rates; }; llvm::Expected diff --git a/compiler/tests/end_to_end_fixture/end_to_end_fhe.yaml b/compiler/tests/end_to_end_fixture/end_to_end_fhe.yaml index a995d7878..066d9d6d0 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_fhe.yaml +++ b/compiler/tests/end_to_end_fixture/end_to_end_fhe.yaml @@ -155,6 +155,13 @@ tests: - scalar: 12 outputs: - scalar: 12 +test-error-rates: + - p-error: 0.05 + nb-repetition: 100 + - p-error: 0.01 + nb-repetition: 100 + - p-error: 0.001 + nb-repetition: 100 --- description: apply_lookup_table_5_bits program: | @@ -218,6 +225,7 @@ tests: - scalar: 96 outputs: - scalar: 96 + --- description: apply_lookup_table_8_bits program: | diff --git a/compiler/tests/end_to_end_fixture/end_to_end_leveled.yaml b/compiler/tests/end_to_end_fixture/end_to_end_leveled.yaml index 74fab6664..bec3279e6 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_leveled.yaml +++ b/compiler/tests/end_to_end_fixture/end_to_end_leveled.yaml @@ -13,6 +13,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_1bits program: | @@ -24,6 +27,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_1bits program: | @@ -37,6 +43,9 @@ tests: - scalar: 0 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_1bits program: | @@ -60,6 +69,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_1_bits program: | @@ -73,6 +85,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_1bits program: | @@ -86,6 +101,9 @@ tests: - scalar: 1 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_1bits program: | @@ -109,6 +127,9 @@ tests: - scalar: 0 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_1bits program: | @@ -122,6 +143,9 @@ tests: - scalar: 1 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_1bits program: | @@ -145,6 +169,9 @@ tests: - scalar: 0 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_1bits program: | @@ -168,6 +195,9 @@ tests: - scalar: 0 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_1bits program: | @@ -185,6 +215,9 @@ tests: - scalar: 0 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_1bits program: | @@ -213,6 +246,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_2bits @@ -225,6 +261,9 @@ tests: - scalar: 3 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_2bits program: | @@ -236,6 +275,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_2bits program: | @@ -249,6 +291,9 @@ tests: - scalar: 2 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_2bits program: | @@ -272,6 +317,9 @@ tests: - scalar: 2 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_2_bits program: | @@ -285,6 +333,9 @@ tests: - scalar: 2 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_2bits program: | @@ -298,6 +349,9 @@ tests: - scalar: 3 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_2bits program: | @@ -321,6 +375,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_2bits program: | @@ -334,6 +391,9 @@ tests: - scalar: 3 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_2bits program: | @@ -357,6 +417,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_2bits program: | @@ -380,6 +443,9 @@ tests: - scalar: 1 outputs: - scalar: 1 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_2bits program: | @@ -397,6 +463,9 @@ tests: - scalar: 1 outputs: - scalar: 2 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_2bits program: | @@ -425,6 +494,9 @@ tests: - scalar: 1 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_3bits @@ -437,6 +509,9 @@ tests: - scalar: 7 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_3bits program: | @@ -448,6 +523,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_3bits program: | @@ -461,6 +539,9 @@ tests: - scalar: 6 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_3bits program: | @@ -484,6 +565,9 @@ tests: - scalar: 4 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_3_bits program: | @@ -497,6 +581,9 @@ tests: - scalar: 4 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_3bits program: | @@ -510,6 +597,9 @@ tests: - scalar: 7 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_3bits program: | @@ -533,6 +623,9 @@ tests: - scalar: 3 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_3bits program: | @@ -546,6 +639,9 @@ tests: - scalar: 7 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_3bits program: | @@ -569,6 +665,9 @@ tests: - scalar: 3 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_3bits program: | @@ -592,6 +691,9 @@ tests: - scalar: 3 outputs: - scalar: 3 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_3bits program: | @@ -609,6 +711,9 @@ tests: - scalar: 3 outputs: - scalar: 6 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_3bits program: | @@ -637,6 +742,9 @@ tests: - scalar: 1 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_4bits @@ -649,6 +757,9 @@ tests: - scalar: 15 outputs: - scalar: 15 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_4bits program: | @@ -660,6 +771,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_4bits program: | @@ -673,6 +787,9 @@ tests: - scalar: 14 outputs: - scalar: 15 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_4bits program: | @@ -696,6 +813,9 @@ tests: - scalar: 8 outputs: - scalar: 15 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_4_bits program: | @@ -709,6 +829,9 @@ tests: - scalar: 8 outputs: - scalar: 15 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_4bits program: | @@ -722,6 +845,9 @@ tests: - scalar: 15 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_4bits program: | @@ -745,6 +871,9 @@ tests: - scalar: 7 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_4bits program: | @@ -758,6 +887,9 @@ tests: - scalar: 15 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_4bits program: | @@ -781,6 +913,9 @@ tests: - scalar: 7 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_4bits program: | @@ -804,6 +939,9 @@ tests: - scalar: 7 outputs: - scalar: 7 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_4bits program: | @@ -821,6 +959,9 @@ tests: - scalar: 7 outputs: - scalar: 14 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_4bits program: | @@ -849,6 +990,9 @@ tests: - scalar: 1 outputs: - scalar: 15 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_5bits @@ -1709,6 +1853,9 @@ tests: - scalar: 511 outputs: - scalar: 511 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_9bits program: | @@ -1720,6 +1867,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_9bits program: | @@ -1733,6 +1883,9 @@ tests: - scalar: 510 outputs: - scalar: 511 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_9bits program: | @@ -1756,6 +1909,9 @@ tests: - scalar: 256 outputs: - scalar: 511 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_9_bits program: | @@ -1769,6 +1925,9 @@ tests: - scalar: 256 outputs: - scalar: 511 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_9bits program: | @@ -1782,6 +1941,9 @@ tests: - scalar: 511 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_9bits program: | @@ -1805,6 +1967,9 @@ tests: - scalar: 255 outputs: - scalar: 255 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_9bits program: | @@ -1818,6 +1983,9 @@ tests: - scalar: 511 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_9bits program: | @@ -1841,6 +2009,9 @@ tests: - scalar: 255 outputs: - scalar: 255 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_9bits program: | @@ -1864,6 +2035,9 @@ tests: - scalar: 255 outputs: - scalar: 255 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_9bits program: | @@ -1881,6 +2055,9 @@ tests: - scalar: 255 outputs: - scalar: 510 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_9bits program: | @@ -1909,6 +2086,9 @@ tests: - scalar: 1 outputs: - scalar: 511 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_10bits @@ -3193,6 +3373,9 @@ tests: - scalar: 65535 outputs: - scalar: 65535 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_16bits program: | @@ -3204,6 +3387,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_16bits program: | @@ -3217,6 +3403,9 @@ tests: - scalar: 65534 outputs: - scalar: 65535 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_16bits program: | @@ -3240,6 +3429,9 @@ tests: - scalar: 32768 outputs: - scalar: 65535 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_16_bits program: | @@ -3253,6 +3445,9 @@ tests: - scalar: 32768 outputs: - scalar: 65535 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_16bits program: | @@ -3266,6 +3461,9 @@ tests: - scalar: 65535 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_16bits program: | @@ -3289,6 +3487,9 @@ tests: - scalar: 32767 outputs: - scalar: 32767 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_16bits program: | @@ -3302,6 +3503,9 @@ tests: - scalar: 65535 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_16bits program: | @@ -3325,6 +3529,9 @@ tests: - scalar: 32767 outputs: - scalar: 32767 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_16bits program: | @@ -3348,6 +3555,9 @@ tests: - scalar: 32767 outputs: - scalar: 32767 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_16bits program: | @@ -3365,6 +3575,9 @@ tests: - scalar: 32767 outputs: - scalar: 65534 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_16bits program: | @@ -3393,6 +3606,9 @@ tests: - scalar: 1 outputs: - scalar: 65535 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_17bits @@ -4889,6 +5105,9 @@ tests: - scalar: 16777215 outputs: - scalar: 16777215 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_24bits program: | @@ -4900,6 +5119,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_24bits program: | @@ -4913,6 +5135,9 @@ tests: - scalar: 16777214 outputs: - scalar: 16777215 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_arg_24bits program: | @@ -4936,6 +5161,9 @@ tests: - scalar: 8388608 outputs: - scalar: 16777215 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_24_bits program: | @@ -4949,6 +5177,9 @@ tests: - scalar: 8388608 outputs: - scalar: 16777215 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_24bits program: | @@ -4962,6 +5193,9 @@ tests: - scalar: 16777215 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_arg_24bits program: | @@ -4985,6 +5219,9 @@ tests: - scalar: 8388607 outputs: - scalar: 8388607 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_24bits program: | @@ -4998,6 +5235,9 @@ tests: - scalar: 16777215 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_arg_24bits program: | @@ -5021,6 +5261,9 @@ tests: - scalar: 8388607 outputs: - scalar: 8388607 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_24bits program: | @@ -5044,6 +5287,9 @@ tests: - scalar: 8388607 outputs: - scalar: 8388607 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_24bits program: | @@ -5061,6 +5307,9 @@ tests: - scalar: 8388607 outputs: - scalar: 16777214 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_arg_24bits program: | @@ -5089,6 +5338,9 @@ tests: - scalar: 1 outputs: - scalar: 16777215 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_25bits @@ -6294,6 +6546,9 @@ tests: - scalar: 4294967295 outputs: - scalar: 4294967295 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_32bits program: | @@ -6305,6 +6560,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_32bits program: | @@ -6318,6 +6576,9 @@ tests: - scalar: 4294967294 outputs: - scalar: 4294967295 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_32_bits program: | @@ -6331,6 +6592,9 @@ tests: - scalar: 2147483648 outputs: - scalar: 4294967295 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_32bits program: | @@ -6344,6 +6608,9 @@ tests: - scalar: 4294967295 outputs: - scalar: 4261412864 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_32bits program: | @@ -6357,6 +6624,9 @@ tests: - scalar: 33554431 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_32bits program: | @@ -6380,6 +6650,9 @@ tests: - scalar: 2147483647 outputs: - scalar: 2147483647 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_32bits program: | @@ -6397,6 +6670,9 @@ tests: - scalar: 2147483647 outputs: - scalar: 4294967294 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- --- description: identity_33bits @@ -9169,6 +9445,9 @@ tests: - scalar: 144115188075855871 outputs: - scalar: 144115188075855871 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: zero_tensor_57bits program: | @@ -9180,6 +9459,9 @@ tests: - outputs: - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] shape: [2,2,4] +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_int_cst_57bits program: | @@ -9193,6 +9475,9 @@ tests: - scalar: 144115188075855870 outputs: - scalar: 144115188075855871 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: add_eint_57_bits program: | @@ -9206,6 +9491,9 @@ tests: - scalar: 72057594037927936 outputs: - scalar: 144115188075855871 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_int_cst_57bits program: | @@ -9219,6 +9507,9 @@ tests: - scalar: 144115188075855871 outputs: - scalar: 144115188075855871 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_int_eint_cst_57bits program: | @@ -9232,6 +9523,9 @@ tests: - scalar: 0 outputs: - scalar: 0 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: sub_eint_57bits program: | @@ -9255,6 +9549,9 @@ tests: - scalar: 72057594037927935 outputs: - scalar: 72057594037927935 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- description: mul_eint_int_cst_57bits program: | @@ -9272,4 +9569,7 @@ tests: - scalar: 72057594037927935 outputs: - scalar: 144115188075855870 +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000 --- diff --git a/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py b/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py index 7051ac7e6..1858415a4 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py +++ b/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py @@ -1,6 +1,15 @@ MIN_PRECISON = 1 MAX_PRECISION = 57 +TEST_ERROR_RATES = """\ +test-error-rates: + - p-error: 0.0001 + nb-repetition: 10000""" + +PRECISIONS_WITH_ERROR_RATES = { + 1, 2, 3, 4, 9, 16, 24, 32, 57 +} + def main(): print("# /!\ DO NOT EDIT MANUALLY THIS FILE MANUALLY") @@ -9,6 +18,9 @@ def main(): for p in range(MIN_PRECISON, MAX_PRECISION+1): if p != 1: print("---") + def may_check_error_rate(): + if p in PRECISIONS_WITH_ERROR_RATES: + print(TEST_ERROR_RATES) max_value = (2 ** p) - 1 integer_bitwidth = p + 1 max_constant = min((2 ** (57-p)) - 1, max_value) @@ -25,6 +37,7 @@ def main(): print(" - scalar: {0}".format(max_value)) print(" outputs:") print(" - scalar: {0}".format(max_value)) + may_check_error_rate() print("---") # zero_tensor print("description: zero_tensor_{0}bits".format(p)) @@ -39,6 +52,7 @@ def main(): print(" - outputs:") print(" - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]") print(" shape: [2,2,4]") + may_check_error_rate() print("---") # add_eint_int_cst print("description: add_eint_int_cst_{0}bits".format(p)) @@ -55,6 +69,7 @@ def main(): print(" - scalar: {0}".format(max_value-1)) print(" outputs:") print(" - scalar: {0}".format(max_value)) + may_check_error_rate() print("---") # add_eint_int_arg if p <= 28: @@ -84,6 +99,7 @@ def main(): print(" - scalar: {0}".format((max_value >> 1) + 1)) print(" outputs:") print(" - scalar: {0}".format(max_value)) + may_check_error_rate() print("---") # add_eint print("description: add_eint_{0}_bits".format(p)) @@ -100,6 +116,7 @@ def main(): print(" - scalar: {0}".format(((2 ** p) >> 1))) print(" outputs:") print(" - scalar: {0}".format((2 ** p) - 1)) + may_check_error_rate() print("---") # sub_eint_int_cst print("description: sub_eint_int_cst_{0}bits".format(p)) @@ -117,6 +134,7 @@ def main(): print(" - scalar: {0}".format(max_value)) print(" outputs:") print(" - scalar: {0}".format(max_value-max_constant)) + may_check_error_rate() print("---") # sub_eint_int_arg if p <= 28: @@ -146,6 +164,7 @@ def main(): print(" - scalar: {0}".format(max_value >> 1)) print(" outputs:") print(" - scalar: {0}".format(max_value >> 1)) + may_check_error_rate() print("---") # sub_int_eint_cst print("description: sub_int_eint_cst_{0}bits".format(p)) @@ -163,6 +182,7 @@ def main(): print(" - scalar: {0}".format(max_constant)) print(" outputs:") print(" - scalar: 0") + may_check_error_rate() print("---") # sub_int_eint_arg if p <= 28: @@ -192,6 +212,7 @@ def main(): print(" - scalar: {0}".format(max_value >> 1)) print(" outputs:") print(" - scalar: {0}".format(max_value >> 1)) + may_check_error_rate() print("---") # sub_eint print("description: sub_eint_{0}bits".format(p)) @@ -218,6 +239,7 @@ def main(): print(" - scalar: {0}".format(max_value >> 1)) print(" outputs:") print(" - scalar: {0}".format(max_value >> 1)) + may_check_error_rate() print("---") # mul_eint_int_cst print("description: mul_eint_int_cst_{0}bits".format(p)) @@ -238,6 +260,7 @@ def main(): print(" - scalar: {0}".format(max_value >> 1)) print(" outputs:") print(" - scalar: {0}".format(max_value - 1)) + may_check_error_rate() print("---") # mul_eint_int_arg if p <= 28: @@ -272,6 +295,7 @@ def main(): print(" - scalar: 1") print(" outputs:") print(" - scalar: {0}".format(max_value)) + may_check_error_rate() print("---") diff --git a/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc b/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc index 137012a30..40367b5eb 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc +++ b/compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc @@ -10,9 +10,18 @@ #include "tests_tools/GtestEnvironment.h" #include "tests_tools/keySetCache.h" +#define CHECK_OR_ERROR(val) \ + { \ + if (!bool(val)) { \ + return StreamStringError(llvm::toString(std::move(val.takeError())) + \ + "\nInvalid '" #val "'"); \ + } \ + } + +using mlir::concretelang::StreamStringError; + template void compile_and_run(EndToEndDesc desc, LambdaSupport support) { - mlir::concretelang::CompilationOptions options("main"); if (desc.v0Constraint.hasValue()) { options.v0FHEConstraints = *desc.v0Constraint; @@ -23,6 +32,20 @@ void compile_and_run(EndToEndDesc desc, LambdaSupport support) { if (desc.largeIntegerParameter.hasValue()) { options.largeIntegerParameter = *desc.largeIntegerParameter; } + if (desc.test_error_rates.empty()) { + compile_and_run_for_config(desc, support, options, llvm::None); + } else { + for (auto test_error_rate : desc.test_error_rates) { + options.optimizerConfig.p_error = test_error_rate.p_error; + compile_and_run_for_config(desc, support, options, test_error_rate); + } + } +} + +template +void compile_and_run_for_config(EndToEndDesc desc, LambdaSupport support, + mlir::concretelang::CompilationOptions options, + llvm::Optional test_error_rate) { /* 0 - Enable parallel testing where required */ #ifdef CONCRETELANG_DATAFLOW_TESTING_ENABLED @@ -47,39 +70,82 @@ void compile_and_run(EndToEndDesc desc, LambdaSupport support) { auto serverLambda = support.loadServerLambda(**compilationResult); ASSERT_EXPECTED_SUCCESS(serverLambda); - /* For each test entries */ - for (auto test : desc.tests) { - std::vector inputArguments; - inputArguments.reserve(test.inputs.size()); - for (auto input : test.inputs) { - auto arg = valueDescriptionToLambdaArgument(input); - ASSERT_EXPECTED_SUCCESS(arg); - inputArguments.push_back(arg.get()); + assert_all_test_entries(desc, test_error_rate, support, keySet, + evaluationKeys, clientParameters, serverLambda); +} + +template +llvm::Error run_once_1_test_entry_once(TestDescription &test, + LambdaSupport &support, KeySet &keySet, + EvaluationKeys &evaluationKeys, + ClientParameters &clientParameters, + ServerLambda &serverLambda) { + std::vector inputArguments; + inputArguments.reserve(test.inputs.size()); + for (auto input : test.inputs) { + auto arg = valueDescriptionToLambdaArgument(input); + CHECK_OR_ERROR(arg); + inputArguments.push_back(arg.get()); + } + + /* 4 - Create the public arguments */ + auto publicArguments = + support.exportArguments(*clientParameters, **keySet, inputArguments); + CHECK_OR_ERROR(publicArguments); + + /* 5 - Call the server lambda */ + auto publicResult = + support.serverCall(*serverLambda, **publicArguments, evaluationKeys); + CHECK_OR_ERROR(publicResult); + + /* 6 - Decrypt the public result */ + auto result = mlir::concretelang::typedResult< + std::unique_ptr>(**keySet, + **publicResult); + + /* 7 - Check result */ + CHECK_OR_ERROR(result); + auto error = checkResult(test.outputs[0], **result); + for (auto arg : inputArguments) { + delete arg; + } + return error; +} + +template +void assert_all_test_entries(EndToEndDesc &desc, + llvm::Optional &opt_test_error_rate, + LambdaSupport &support, KeySet &keySet, + EvaluationKeys &evaluationKeys, + ClientParameters &clientParameters, + ServerLambda &serverLambda) { + auto run = [&](TestDescription &test) { + return run_once_1_test_entry_once(test, support, keySet, evaluationKeys, + clientParameters, serverLambda); + }; + if (!opt_test_error_rate.has_value()) { + for (auto test : desc.tests) { + ASSERT_LLVM_ERROR(run(test)); } - - /* 4 - Create the public arguments */ - auto publicArguments = - support.exportArguments(*clientParameters, **keySet, inputArguments); - ASSERT_EXPECTED_SUCCESS(publicArguments); - - /* 5 - Call the server lambda */ - auto publicResult = - support.serverCall(*serverLambda, **publicArguments, evaluationKeys); - ASSERT_EXPECTED_SUCCESS(publicResult); - - /* 6 - Decrypt the public result */ - auto result = mlir::concretelang::typedResult< - std::unique_ptr>(**keySet, - **publicResult); - - /* 7 - Check result */ - ASSERT_EXPECTED_SUCCESS(result); - ASSERT_LLVM_ERROR(checkResult(test.outputs[0], **result)); - - for (auto arg : inputArguments) { - delete arg; + return; + } + auto test_error_rate = opt_test_error_rate.value(); + ASSERT_LE(desc.tests.size(), test_error_rate.nb_repetition); + int nb_error = 0; + for (size_t i = 0; i < test_error_rate.nb_repetition; i++) { + auto test = desc.tests[i % desc.tests.size()]; + auto error = run(test); + if (error) { + nb_error += 1; + DISCARD_LLVM_ERROR(error); } } + double maximum_errors = test_error_rate.too_high_error_count_threshold(); + // std::cout << "n_rep " << maximum_errors << " p_error " << + // test_error_rate.p_error << " maximum_errors " << maximum_errors << "\n"; + ASSERT_LE(nb_error, maximum_errors) << "Empirical error rate is too high"; } std::string printEndToEndDesc(const testing::TestParamInfo desc) { diff --git a/compiler/tests/tests_tools/assert.h b/compiler/tests/tests_tools/assert.h index 14045db0d..9f25e8428 100644 --- a/compiler/tests/tests_tools/assert.h +++ b/compiler/tests/tests_tools/assert.h @@ -15,6 +15,16 @@ } \ } +#define DISCARD_LLVM_ERROR(err) \ + { \ + llvm::Error e = std::move(err); \ + if (e) { \ + handleAllErrors(std::move(e), [](const llvm::ErrorInfoBase &ei) { \ + ASSERT_TRUE(true); \ + }); \ + } \ + } + // Checks that the value `val` is not in an error state. Returns // `true` if the test passes, otherwise `false`. template