feat(test): check test error rate for fast cases

compiler/tests/end_to_end_fixture/EndToEndFixture.cpp

@@ -1,3 +1,5 @@
+#include <cmath>
+
 #include "EndToEndFixture.h"
 #include "concretelang/Support/CompilerEngine.h"
 #include "concretelang/Support/Jit.h"
@@ -6,6 +8,61 @@
 
 using mlir::concretelang::StreamStringError;
 
+// derived from https://stackoverflow.com/a/45869209
+uint64_t solve_binomial_cdf_bigger_than(size_t n, double p_error,
+                                        double p_mass) {
+  // Solve: find k such that
+  //   - binomial_cdf(k, n, p_error) >= p_mass
+  //   - given n and p_error and p_mass
+  // Notation:
+  //   - n is the number of repetition,
+  //   - p_error the probability of error,
+  //   - p_mass is the mass probability of having a number of error <= k
+  //   - k is a number of error that is returned
+  // This returns the smallest threshold for error such that the number of error
+  // is below or equal, with probability p_mass.
+  using std::exp;
+  using std::log;
+  auto log_p_error = log(p_error);
+  auto log_p_success = log(1.0 - p_error);
+  auto cdf_k = 0.0;
+  double log_pmf_k;
+  // k in 0..n, isum and stop when the cumulative attained p_mass
+  for (uint64_t k = 0; k < n; k++) {
+    if (k == 0) {
+      // start with n success
+      log_pmf_k = n * log_p_success;
+    } else {
+      // add one error and remove one success
+      log_pmf_k += log(n - k + 1) - log(k) + (log_p_error - log_p_success);
+    }
+    cdf_k += exp(log_pmf_k);
+    if (cdf_k > p_mass) {
+      return k;
+    }
+  }
+  return n;
+}
+
+uint64_t TestErrorRate::too_high_error_count_threshold() {
+  // Return the smallest threshold for error such that
+  // the number of error is higher than this threshold
+  // with probability FALSE_ALARM_RATE.
+
+  // Examples:
+  // n_rep=100, p_error=0.05 -> 16 error max  (ratio 3)
+  // n_rep=100, p_error=0.01 -> 7 error max (ratio 7)
+  // n_rep=100, p_error=0.001 -> 3 error max (ratio 30)
+  // n_rep=10000, p_error=0.0001 -> 8 error max (ratio 8)
+  // A high ratio indicate that the number of repetition should be increased.
+  // A good ratio (but costly) is between 1 and 2.
+  // A bad ratio can still detect most issues.
+  // A good ratio will help to detect more precise calibration issue.
+  double p_mass = 1.0 - TestErrorRate::FALSE_ALARM_RATE;
+  return solve_binomial_cdf_bigger_than(this->nb_repetition, this->p_error,
+                                        p_mass);
+}
+
 llvm::Expected<mlir::concretelang::LambdaArgument *>
 scalarDescToLambdaArgument(ScalarDesc desc) {
   switch (desc.width) {
@@ -176,6 +233,15 @@ template <> struct llvm::yaml::MappingTraits<TestDescription> {
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(TestDescription)
 
+template <> struct llvm::yaml::MappingTraits<TestErrorRate> {
+  static void mapping(IO &io, TestErrorRate &desc) {
+    io.mapRequired("p-error", desc.p_error);
+    io.mapRequired("nb-repetition", desc.nb_repetition);
+  }
+};
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(TestErrorRate)
+
 template <> struct llvm::yaml::MappingTraits<EndToEndDesc> {
   static void mapping(IO &io, EndToEndDesc &desc) {
     io.mapRequired("description", desc.description);
@@ -210,6 +276,7 @@ template <> struct llvm::yaml::MappingTraits<EndToEndDesc> {
     if (!largeInterger.crtDecomposition.empty()) {
       desc.largeIntegerParameter = largeInterger;
     }
+    io.mapOptional("test-error-rates", desc.test_error_rates);
   }
 };
 
@@ -227,7 +294,9 @@ std::vector<EndToEndDesc> loadEndToEndDesc(std::string path) {
   yin >> desc;
 
   // Check for error
-  if (yin.error())
-    assert(false && "cannot parse doc");
+  if (yin.error()) { // .error() displays the beginning of the error message
+    std::cerr << "In yaml file: " << path << "\n";
+    assert(false);
+  }
   return desc;
 }

compiler/tests/end_to_end_fixture/EndToEndFixture.h

@@ -37,6 +37,14 @@ struct TestDescription {
   std::vector<ValueDescription> outputs;
 };
 
+struct TestErrorRate {
+  double p_error;
+  uint64_t nb_repetition;
+  // rate at which a valid code will make the test fail due to bad luck
+  const double FALSE_ALARM_RATE = 0.00001;
+  uint64_t too_high_error_count_threshold();
+};
+
 struct EndToEndDesc {
   std::string description;
   std::string program;
@@ -45,6 +53,7 @@ struct EndToEndDesc {
   llvm::Optional<mlir::concretelang::V0FHEConstraint> v0Constraint;
   llvm::Optional<mlir::concretelang::LargeIntegerParameter>
       largeIntegerParameter;
+  std::vector<TestErrorRate> test_error_rates;
 };
 
 llvm::Expected<mlir::concretelang::LambdaArgument *>

compiler/tests/end_to_end_fixture/end_to_end_fhe.yaml

@@ -155,6 +155,13 @@ tests:
     - scalar: 12
     outputs:
     - scalar: 12
+test-error-rates:
+  - p-error: 0.05
+    nb-repetition: 100
+  - p-error: 0.01
+    nb-repetition: 100
+  - p-error: 0.001
+    nb-repetition: 100
 ---
 description: apply_lookup_table_5_bits
 program: |
@@ -218,6 +225,7 @@ tests:
     - scalar: 96
     outputs:
     - scalar: 96
+
 ---
 description: apply_lookup_table_8_bits
 program: |

compiler/tests/end_to_end_fixture/end_to_end_leveled.yaml

@@ -13,6 +13,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_1bits
 program: |
@@ -24,6 +27,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_1bits
 program: |
@@ -37,6 +43,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_1bits
 program: |
@@ -60,6 +69,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_1_bits
 program: |
@@ -73,6 +85,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_1bits
 program: |
@@ -86,6 +101,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_1bits
 program: |
@@ -109,6 +127,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_1bits
 program: |
@@ -122,6 +143,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_1bits
 program: |
@@ -145,6 +169,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_1bits
 program: |
@@ -168,6 +195,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_1bits
 program: |
@@ -185,6 +215,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_1bits
 program: |
@@ -213,6 +246,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_2bits
@@ -225,6 +261,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_2bits
 program: |
@@ -236,6 +275,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_2bits
 program: |
@@ -249,6 +291,9 @@ tests:
     - scalar: 2
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_2bits
 program: |
@@ -272,6 +317,9 @@ tests:
     - scalar: 2
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_2_bits
 program: |
@@ -285,6 +333,9 @@ tests:
     - scalar: 2
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_2bits
 program: |
@@ -298,6 +349,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_2bits
 program: |
@@ -321,6 +375,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_2bits
 program: |
@@ -334,6 +391,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_2bits
 program: |
@@ -357,6 +417,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_2bits
 program: |
@@ -380,6 +443,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 1
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_2bits
 program: |
@@ -397,6 +463,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 2
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_2bits
 program: |
@@ -425,6 +494,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_3bits
@@ -437,6 +509,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_3bits
 program: |
@@ -448,6 +523,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_3bits
 program: |
@@ -461,6 +539,9 @@ tests:
     - scalar: 6
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_3bits
 program: |
@@ -484,6 +565,9 @@ tests:
     - scalar: 4
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_3_bits
 program: |
@@ -497,6 +581,9 @@ tests:
     - scalar: 4
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_3bits
 program: |
@@ -510,6 +597,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_3bits
 program: |
@@ -533,6 +623,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_3bits
 program: |
@@ -546,6 +639,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_3bits
 program: |
@@ -569,6 +665,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_3bits
 program: |
@@ -592,6 +691,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 3
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_3bits
 program: |
@@ -609,6 +711,9 @@ tests:
     - scalar: 3
     outputs:
     - scalar: 6
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_3bits
 program: |
@@ -637,6 +742,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_4bits
@@ -649,6 +757,9 @@ tests:
     - scalar: 15
     outputs:
     - scalar: 15
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_4bits
 program: |
@@ -660,6 +771,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_4bits
 program: |
@@ -673,6 +787,9 @@ tests:
     - scalar: 14
     outputs:
     - scalar: 15
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_4bits
 program: |
@@ -696,6 +813,9 @@ tests:
     - scalar: 8
     outputs:
     - scalar: 15
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_4_bits
 program: |
@@ -709,6 +829,9 @@ tests:
     - scalar: 8
     outputs:
     - scalar: 15
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_4bits
 program: |
@@ -722,6 +845,9 @@ tests:
     - scalar: 15
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_4bits
 program: |
@@ -745,6 +871,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_4bits
 program: |
@@ -758,6 +887,9 @@ tests:
     - scalar: 15
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_4bits
 program: |
@@ -781,6 +913,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_4bits
 program: |
@@ -804,6 +939,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 7
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_4bits
 program: |
@@ -821,6 +959,9 @@ tests:
     - scalar: 7
     outputs:
     - scalar: 14
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_4bits
 program: |
@@ -849,6 +990,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 15
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_5bits
@@ -1709,6 +1853,9 @@ tests:
     - scalar: 511
     outputs:
     - scalar: 511
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_9bits
 program: |
@@ -1720,6 +1867,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_9bits
 program: |
@@ -1733,6 +1883,9 @@ tests:
     - scalar: 510
     outputs:
     - scalar: 511
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_9bits
 program: |
@@ -1756,6 +1909,9 @@ tests:
     - scalar: 256
     outputs:
     - scalar: 511
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_9_bits
 program: |
@@ -1769,6 +1925,9 @@ tests:
     - scalar: 256
     outputs:
     - scalar: 511
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_9bits
 program: |
@@ -1782,6 +1941,9 @@ tests:
     - scalar: 511
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_9bits
 program: |
@@ -1805,6 +1967,9 @@ tests:
     - scalar: 255
     outputs:
     - scalar: 255
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_9bits
 program: |
@@ -1818,6 +1983,9 @@ tests:
     - scalar: 511
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_9bits
 program: |
@@ -1841,6 +2009,9 @@ tests:
     - scalar: 255
     outputs:
     - scalar: 255
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_9bits
 program: |
@@ -1864,6 +2035,9 @@ tests:
     - scalar: 255
     outputs:
     - scalar: 255
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_9bits
 program: |
@@ -1881,6 +2055,9 @@ tests:
     - scalar: 255
     outputs:
     - scalar: 510
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_9bits
 program: |
@@ -1909,6 +2086,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 511
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_10bits
@@ -3193,6 +3373,9 @@ tests:
     - scalar: 65535
     outputs:
     - scalar: 65535
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_16bits
 program: |
@@ -3204,6 +3387,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_16bits
 program: |
@@ -3217,6 +3403,9 @@ tests:
     - scalar: 65534
     outputs:
     - scalar: 65535
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_16bits
 program: |
@@ -3240,6 +3429,9 @@ tests:
     - scalar: 32768
     outputs:
     - scalar: 65535
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_16_bits
 program: |
@@ -3253,6 +3445,9 @@ tests:
     - scalar: 32768
     outputs:
     - scalar: 65535
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_16bits
 program: |
@@ -3266,6 +3461,9 @@ tests:
     - scalar: 65535
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_16bits
 program: |
@@ -3289,6 +3487,9 @@ tests:
     - scalar: 32767
     outputs:
     - scalar: 32767
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_16bits
 program: |
@@ -3302,6 +3503,9 @@ tests:
     - scalar: 65535
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_16bits
 program: |
@@ -3325,6 +3529,9 @@ tests:
     - scalar: 32767
     outputs:
     - scalar: 32767
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_16bits
 program: |
@@ -3348,6 +3555,9 @@ tests:
     - scalar: 32767
     outputs:
     - scalar: 32767
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_16bits
 program: |
@@ -3365,6 +3575,9 @@ tests:
     - scalar: 32767
     outputs:
     - scalar: 65534
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_16bits
 program: |
@@ -3393,6 +3606,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 65535
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_17bits
@@ -4889,6 +5105,9 @@ tests:
     - scalar: 16777215
     outputs:
     - scalar: 16777215
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_24bits
 program: |
@@ -4900,6 +5119,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_24bits
 program: |
@@ -4913,6 +5135,9 @@ tests:
     - scalar: 16777214
     outputs:
     - scalar: 16777215
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_arg_24bits
 program: |
@@ -4936,6 +5161,9 @@ tests:
     - scalar: 8388608
     outputs:
     - scalar: 16777215
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_24_bits
 program: |
@@ -4949,6 +5177,9 @@ tests:
     - scalar: 8388608
     outputs:
     - scalar: 16777215
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_24bits
 program: |
@@ -4962,6 +5193,9 @@ tests:
     - scalar: 16777215
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_arg_24bits
 program: |
@@ -4985,6 +5219,9 @@ tests:
     - scalar: 8388607
     outputs:
     - scalar: 8388607
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_24bits
 program: |
@@ -4998,6 +5235,9 @@ tests:
     - scalar: 16777215
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_arg_24bits
 program: |
@@ -5021,6 +5261,9 @@ tests:
     - scalar: 8388607
     outputs:
     - scalar: 8388607
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_24bits
 program: |
@@ -5044,6 +5287,9 @@ tests:
     - scalar: 8388607
     outputs:
     - scalar: 8388607
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_24bits
 program: |
@@ -5061,6 +5307,9 @@ tests:
     - scalar: 8388607
     outputs:
     - scalar: 16777214
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_arg_24bits
 program: |
@@ -5089,6 +5338,9 @@ tests:
     - scalar: 1
     outputs:
     - scalar: 16777215
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_25bits
@@ -6294,6 +6546,9 @@ tests:
     - scalar: 4294967295
     outputs:
     - scalar: 4294967295
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_32bits
 program: |
@@ -6305,6 +6560,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_32bits
 program: |
@@ -6318,6 +6576,9 @@ tests:
     - scalar: 4294967294
     outputs:
     - scalar: 4294967295
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_32_bits
 program: |
@@ -6331,6 +6592,9 @@ tests:
     - scalar: 2147483648
     outputs:
     - scalar: 4294967295
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_32bits
 program: |
@@ -6344,6 +6608,9 @@ tests:
     - scalar: 4294967295
     outputs:
     - scalar: 4261412864
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_32bits
 program: |
@@ -6357,6 +6624,9 @@ tests:
     - scalar: 33554431
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_32bits
 program: |
@@ -6380,6 +6650,9 @@ tests:
     - scalar: 2147483647
     outputs:
     - scalar: 2147483647
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_32bits
 program: |
@@ -6397,6 +6670,9 @@ tests:
     - scalar: 2147483647
     outputs:
     - scalar: 4294967294
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 ---
 description: identity_33bits
@@ -9169,6 +9445,9 @@ tests:
     - scalar: 144115188075855871
     outputs:
     - scalar: 144115188075855871
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: zero_tensor_57bits
 program: |
@@ -9180,6 +9459,9 @@ tests:
   - outputs:
     - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
       shape: [2,2,4]
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_int_cst_57bits
 program: |
@@ -9193,6 +9475,9 @@ tests:
     - scalar: 144115188075855870
     outputs:
     - scalar: 144115188075855871
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: add_eint_57_bits
 program: |
@@ -9206,6 +9491,9 @@ tests:
     - scalar: 72057594037927936
     outputs:
     - scalar: 144115188075855871
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_int_cst_57bits
 program: |
@@ -9219,6 +9507,9 @@ tests:
     - scalar: 144115188075855871
     outputs:
     - scalar: 144115188075855871
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_int_eint_cst_57bits
 program: |
@@ -9232,6 +9523,9 @@ tests:
     - scalar: 0
     outputs:
     - scalar: 0
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: sub_eint_57bits
 program: |
@@ -9255,6 +9549,9 @@ tests:
     - scalar: 72057594037927935
     outputs:
     - scalar: 72057594037927935
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---
 description: mul_eint_int_cst_57bits
 program: |
@@ -9272,4 +9569,7 @@ tests:
     - scalar: 72057594037927935
     outputs:
     - scalar: 144115188075855870
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000
 ---

compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py

@@ -1,6 +1,15 @@
 MIN_PRECISON = 1
 MAX_PRECISION = 57
 
+TEST_ERROR_RATES = """\
+test-error-rates:
+  - p-error: 0.0001
+    nb-repetition: 10000"""
+
+PRECISIONS_WITH_ERROR_RATES = {
+    1, 2, 3, 4, 9, 16, 24, 32, 57
+}
+
 
 def main():
     print("# /!\ DO NOT EDIT MANUALLY THIS FILE MANUALLY")
@@ -9,6 +18,9 @@ def main():
     for p in range(MIN_PRECISON, MAX_PRECISION+1):
         if p != 1:
             print("---")
+        def may_check_error_rate():
+            if p in PRECISIONS_WITH_ERROR_RATES:
+                print(TEST_ERROR_RATES)
         max_value = (2 ** p) - 1
         integer_bitwidth = p + 1
         max_constant = min((2 ** (57-p)) - 1, max_value)
@@ -25,6 +37,7 @@ def main():
         print("    - scalar: {0}".format(max_value))
         print("    outputs:")
         print("    - scalar: {0}".format(max_value))
+        may_check_error_rate()
         print("---")
         # zero_tensor
         print("description: zero_tensor_{0}bits".format(p))
@@ -39,6 +52,7 @@ def main():
         print("  - outputs:")
         print("    - tensor: [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]")
         print("      shape: [2,2,4]")
+        may_check_error_rate()
         print("---")
         # add_eint_int_cst
         print("description: add_eint_int_cst_{0}bits".format(p))
@@ -55,6 +69,7 @@ def main():
         print("    - scalar: {0}".format(max_value-1))
         print("    outputs:")
         print("    - scalar: {0}".format(max_value))
+        may_check_error_rate()
         print("---")
         # add_eint_int_arg
         if p <= 28:
@@ -84,6 +99,7 @@ def main():
             print("    - scalar: {0}".format((max_value >> 1) + 1))
             print("    outputs:")
             print("    - scalar: {0}".format(max_value))
+            may_check_error_rate()
             print("---")
         # add_eint
         print("description: add_eint_{0}_bits".format(p))
@@ -100,6 +116,7 @@ def main():
         print("    - scalar: {0}".format(((2 ** p) >> 1)))
         print("    outputs:")
         print("    - scalar: {0}".format((2 ** p) - 1))
+        may_check_error_rate()
         print("---")
         # sub_eint_int_cst
         print("description: sub_eint_int_cst_{0}bits".format(p))
@@ -117,6 +134,7 @@ def main():
         print("    - scalar: {0}".format(max_value))
         print("    outputs:")
         print("    - scalar: {0}".format(max_value-max_constant))
+        may_check_error_rate()
         print("---")
         # sub_eint_int_arg
         if p <= 28:
@@ -146,6 +164,7 @@ def main():
             print("    - scalar: {0}".format(max_value >> 1))
             print("    outputs:")
             print("    - scalar: {0}".format(max_value >> 1))
+            may_check_error_rate()
             print("---")
         # sub_int_eint_cst
         print("description: sub_int_eint_cst_{0}bits".format(p))
@@ -163,6 +182,7 @@ def main():
         print("    - scalar: {0}".format(max_constant))
         print("    outputs:")
         print("    - scalar: 0")
+        may_check_error_rate()
         print("---")
         # sub_int_eint_arg
         if p <= 28:
@@ -192,6 +212,7 @@ def main():
             print("    - scalar: {0}".format(max_value >> 1))
             print("    outputs:")
             print("    - scalar: {0}".format(max_value >> 1))
+            may_check_error_rate()
             print("---")
         # sub_eint
         print("description: sub_eint_{0}bits".format(p))
@@ -218,6 +239,7 @@ def main():
         print("    - scalar: {0}".format(max_value >> 1))
         print("    outputs:")
         print("    - scalar: {0}".format(max_value >> 1))
+        may_check_error_rate()
         print("---")
         # mul_eint_int_cst
         print("description: mul_eint_int_cst_{0}bits".format(p))
@@ -238,6 +260,7 @@ def main():
         print("    - scalar: {0}".format(max_value >> 1))
         print("    outputs:")
         print("    - scalar: {0}".format(max_value - 1))
+        may_check_error_rate()
         print("---")
         # mul_eint_int_arg
         if p <= 28:
@@ -272,6 +295,7 @@ def main():
             print("    - scalar: 1")
             print("    outputs:")
             print("    - scalar: {0}".format(max_value))
+            may_check_error_rate()
             print("---")
 
 

compiler/tests/end_to_end_tests/end_to_end_jit_fhe.cc

@@ -10,9 +10,18 @@
 #include "tests_tools/GtestEnvironment.h"
 #include "tests_tools/keySetCache.h"
 
+#define CHECK_OR_ERROR(val)                                                    \
+  {                                                                            \
+    if (!bool(val)) {                                                          \
+      return StreamStringError(llvm::toString(std::move(val.takeError())) +    \
+                               "\nInvalid '" #val "'");                        \
+    }                                                                          \
+  }
+
+using mlir::concretelang::StreamStringError;
+
 template <typename LambdaSupport>
 void compile_and_run(EndToEndDesc desc, LambdaSupport support) {
-
   mlir::concretelang::CompilationOptions options("main");
   if (desc.v0Constraint.hasValue()) {
     options.v0FHEConstraints = *desc.v0Constraint;
@@ -23,6 +32,20 @@ void compile_and_run(EndToEndDesc desc, LambdaSupport support) {
   if (desc.largeIntegerParameter.hasValue()) {
     options.largeIntegerParameter = *desc.largeIntegerParameter;
   }
+  if (desc.test_error_rates.empty()) {
+    compile_and_run_for_config(desc, support, options, llvm::None);
+  } else {
+    for (auto test_error_rate : desc.test_error_rates) {
+      options.optimizerConfig.p_error = test_error_rate.p_error;
+      compile_and_run_for_config(desc, support, options, test_error_rate);
+    }
+  }
+}
+
+template <typename LambdaSupport>
+void compile_and_run_for_config(EndToEndDesc desc, LambdaSupport support,
+                                mlir::concretelang::CompilationOptions options,
+                                llvm::Optional<TestErrorRate> test_error_rate) {
 
   /* 0 - Enable parallel testing where required */
 #ifdef CONCRETELANG_DATAFLOW_TESTING_ENABLED
@@ -47,39 +70,82 @@ void compile_and_run(EndToEndDesc desc, LambdaSupport support) {
   auto serverLambda = support.loadServerLambda(**compilationResult);
   ASSERT_EXPECTED_SUCCESS(serverLambda);
 
-  /* For each test entries */
-  for (auto test : desc.tests) {
-    std::vector<mlir::concretelang::LambdaArgument *> inputArguments;
-    inputArguments.reserve(test.inputs.size());
-    for (auto input : test.inputs) {
-      auto arg = valueDescriptionToLambdaArgument(input);
-      ASSERT_EXPECTED_SUCCESS(arg);
-      inputArguments.push_back(arg.get());
+  assert_all_test_entries(desc, test_error_rate, support, keySet,
+                          evaluationKeys, clientParameters, serverLambda);
+}
+
+template <typename LambdaSupport, typename KeySet, typename EvaluationKeys,
+          typename ClientParameters, typename ServerLambda>
+llvm::Error run_once_1_test_entry_once(TestDescription &test,
+                                       LambdaSupport &support, KeySet &keySet,
+                                       EvaluationKeys &evaluationKeys,
+                                       ClientParameters &clientParameters,
+                                       ServerLambda &serverLambda) {
+  std::vector<mlir::concretelang::LambdaArgument *> inputArguments;
+  inputArguments.reserve(test.inputs.size());
+  for (auto input : test.inputs) {
+    auto arg = valueDescriptionToLambdaArgument(input);
+    CHECK_OR_ERROR(arg);
+    inputArguments.push_back(arg.get());
+  }
+
+  /* 4 - Create the public arguments */
+  auto publicArguments =
+      support.exportArguments(*clientParameters, **keySet, inputArguments);
+  CHECK_OR_ERROR(publicArguments);
+
+  /* 5 - Call the server lambda */
+  auto publicResult =
+      support.serverCall(*serverLambda, **publicArguments, evaluationKeys);
+  CHECK_OR_ERROR(publicResult);
+
+  /* 6 - Decrypt the public result */
+  auto result = mlir::concretelang::typedResult<
+      std::unique_ptr<mlir::concretelang::LambdaArgument>>(**keySet,
+                                                           **publicResult);
+
+  /* 7 - Check result */
+  CHECK_OR_ERROR(result);
+  auto error = checkResult(test.outputs[0], **result);
+  for (auto arg : inputArguments) {
+    delete arg;
+  }
+  return error;
+}
+
+template <typename LambdaSupport, typename KeySet, typename EvaluationKeys,
+          typename ClientParameters, typename ServerLambda>
+void assert_all_test_entries(EndToEndDesc &desc,
+                             llvm::Optional<TestErrorRate> &opt_test_error_rate,
+                             LambdaSupport &support, KeySet &keySet,
+                             EvaluationKeys &evaluationKeys,
+                             ClientParameters &clientParameters,
+                             ServerLambda &serverLambda) {
+  auto run = [&](TestDescription &test) {
+    return run_once_1_test_entry_once(test, support, keySet, evaluationKeys,
+                                      clientParameters, serverLambda);
+  };
+  if (!opt_test_error_rate.has_value()) {
+    for (auto test : desc.tests) {
+      ASSERT_LLVM_ERROR(run(test));
     }
-
-    /* 4 - Create the public arguments */
-    auto publicArguments =
-        support.exportArguments(*clientParameters, **keySet, inputArguments);
-    ASSERT_EXPECTED_SUCCESS(publicArguments);
-
-    /* 5 - Call the server lambda */
-    auto publicResult =
-        support.serverCall(*serverLambda, **publicArguments, evaluationKeys);
-    ASSERT_EXPECTED_SUCCESS(publicResult);
-
-    /* 6 - Decrypt the public result */
-    auto result = mlir::concretelang::typedResult<
-        std::unique_ptr<mlir::concretelang::LambdaArgument>>(**keySet,
-                                                             **publicResult);
-
-    /* 7 - Check result */
-    ASSERT_EXPECTED_SUCCESS(result);
-    ASSERT_LLVM_ERROR(checkResult(test.outputs[0], **result));
-
-    for (auto arg : inputArguments) {
-      delete arg;
+    return;
+  }
+  auto test_error_rate = opt_test_error_rate.value();
+  ASSERT_LE(desc.tests.size(), test_error_rate.nb_repetition);
+  int nb_error = 0;
+  for (size_t i = 0; i < test_error_rate.nb_repetition; i++) {
+    auto test = desc.tests[i % desc.tests.size()];
+    auto error = run(test);
+    if (error) {
+      nb_error += 1;
+      DISCARD_LLVM_ERROR(error);
     }
   }
+  double maximum_errors = test_error_rate.too_high_error_count_threshold();
+  // std::cout << "n_rep " << maximum_errors << " p_error " <<
+  // test_error_rate.p_error <<  " maximum_errors " << maximum_errors << "\n";
+  ASSERT_LE(nb_error, maximum_errors) << "Empirical error rate is too high";
 }
 
 std::string printEndToEndDesc(const testing::TestParamInfo<EndToEndDesc> desc) {

compiler/tests/tests_tools/assert.h

@@ -15,6 +15,16 @@
     }                                                                          \
   }
 
+#define DISCARD_LLVM_ERROR(err)                                                \
+  {                                                                            \
+    llvm::Error e = std::move(err);                                            \
+    if (e) {                                                                   \
+      handleAllErrors(std::move(e), [](const llvm::ErrorInfoBase &ei) {        \
+        ASSERT_TRUE(true);                                                     \
+      });                                                                      \
+    }                                                                          \
+  }
+
 // Checks that the value `val` is not in an error state. Returns
 // `true` if the test passes, otherwise `false`.
 template <typename T>