From ed2adceafa82f665d345b920f68d7eec57c2c74f Mon Sep 17 00:00:00 2001 From: rudy Date: Tue, 6 Dec 2022 10:45:23 +0100 Subject: [PATCH] test: tensorized test/bench rely on p-error --- .../end_to_end_benchmark.cpp | 4 +++ .../end_to_end_fixture/EndToEndFixture.cpp | 2 +- .../end_to_end_fixture/EndToEndFixture.h | 1 + .../end_to_end_apply_lookup_table_gen.py | 27 ++++++++++--------- .../end_to_end_leveled_gen.py | 8 ++++-- ...nd_to_end_linalg_apply_lookup_table_gen.py | 12 ++++++--- .../end_to_end_linalg_leveled_gen.py | 8 ++++-- .../end_to_end_tests/end_to_end_jit_test.h | 5 ++-- .../tests/end_to_end_tests/end_to_end_test.cc | 4 +++ .../tests/end_to_end_tests/end_to_end_test.h | 2 ++ 10 files changed, 50 insertions(+), 23 deletions(-) diff --git a/compiler/tests/end_to_end_benchmarks/end_to_end_benchmark.cpp b/compiler/tests/end_to_end_benchmarks/end_to_end_benchmark.cpp index c6319552a..744c77851 100644 --- a/compiler/tests/end_to_end_benchmarks/end_to_end_benchmark.cpp +++ b/compiler/tests/end_to_end_benchmarks/end_to_end_benchmark.cpp @@ -133,6 +133,10 @@ void registerEndToEndBenchmark(std::string suiteName, auto optionsName = getOptionsName(options); for (auto description : descriptions) { options.clientParametersFuncName = "main"; + if (description.p_error) { + assert(std::isnan(options.optimizerConfig.global_p_error)); + options.optimizerConfig.p_error = description.p_error.value(); + } mlir::concretelang::JITSupport support; auto benchName = [&](std::string name) { std::ostringstream s; diff --git a/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp b/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp index 96931ff96..30c49c095 100644 --- a/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp +++ b/compiler/tests/end_to_end_fixture/EndToEndFixture.cpp @@ -305,8 +305,8 @@ template <> struct llvm::yaml::MappingTraits { static void mapping(IO &io, EndToEndDesc &desc) { io.mapRequired("description", desc.description); io.mapRequired("program", desc.program); + io.mapOptional("p-error", desc.p_error); io.mapRequired("tests", desc.tests); - bool use_default_fhe_constraints = false; io.mapOptional("use_default_fhe_constraints", use_default_fhe_constraints); diff --git a/compiler/tests/end_to_end_fixture/EndToEndFixture.h b/compiler/tests/end_to_end_fixture/EndToEndFixture.h index 771e85c7b..089cb8bad 100644 --- a/compiler/tests/end_to_end_fixture/EndToEndFixture.h +++ b/compiler/tests/end_to_end_fixture/EndToEndFixture.h @@ -48,6 +48,7 @@ struct TestErrorRate { struct EndToEndDesc { std::string description; std::string program; + llvm::Optional p_error; // force the test in local p-error std::vector tests; llvm::Optional v0Parameter; llvm::Optional v0Constraint; diff --git a/compiler/tests/end_to_end_fixture/end_to_end_apply_lookup_table_gen.py b/compiler/tests/end_to_end_fixture/end_to_end_apply_lookup_table_gen.py index 46178cc8f..73d7935c0 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_apply_lookup_table_gen.py +++ b/compiler/tests/end_to_end_fixture/end_to_end_apply_lookup_table_gen.py @@ -1,10 +1,9 @@ -from platform import mac_ver -import numpy as np import argparse +from platform import mac_ver +import numpy as np -MIN_PRECISON = 1 -MAX_PRECISION = 16 +from end_to_end_linalg_leveled_gen import P_ERROR PRECISION_FORCE_CRT = 9 @@ -26,6 +25,7 @@ def generate(args): print(" }") if p >= PRECISION_FORCE_CRT: print("encoding: crt") + print(f"p-error: {P_ERROR}") print("tests:") print(" - inputs:") print(" - scalar: 0") @@ -42,12 +42,13 @@ def generate(args): print(f" - scalar: {random_lut[max_value]}") print("---") -CLI = argparse.ArgumentParser() -CLI.add_argument( - "--bitwidth", - help="Specify the list of bitwidth to generate", - nargs="+", - type=int, - default=list(range(1,16)), -) -generate(CLI.parse_args()) +if __name__ == "__main__": + CLI = argparse.ArgumentParser() + CLI.add_argument( + "--bitwidth", + help="Specify the list of bitwidth to generate", + nargs="+", + type=int, + default=list(range(1,17)), + ) + generate(CLI.parse_args()) diff --git a/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py b/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py index 3d93835c3..b45126d7e 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py +++ b/compiler/tests/end_to_end_fixture/end_to_end_leveled_gen.py @@ -1,4 +1,6 @@ MIN_PRECISON = 1 +from end_to_end_linalg_leveled_gen import P_ERROR + MAX_PRECISION = 57 TEST_ERROR_RATES = """\ @@ -251,6 +253,8 @@ def main(): " %1 = \"FHE.mul_eint_int\"(%arg0, %0): (!FHE.eint<{0}>, i{1}) -> (!FHE.eint<{0}>)".format(p, integer_bitwidth)) print(" return %1: !FHE.eint<{0}>".format(p)) print(" }") + if p <= 57: + print(f"p-error: {P_ERROR}") print("tests:") print(" - inputs:") print(" - scalar: 0") @@ -298,5 +302,5 @@ def main(): may_check_error_rate() print("---") - -main() +if __name__ == "__main__": + main() diff --git a/compiler/tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py b/compiler/tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py index 8cf4b4d72..89dcc6856 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py +++ b/compiler/tests/end_to_end_fixture/end_to_end_linalg_apply_lookup_table_gen.py @@ -1,6 +1,9 @@ -from platform import mac_ver -import numpy as np import argparse +from platform import mac_ver + +import numpy as np + +from end_to_end_linalg_leveled_gen import P_ERROR PRECISION_FORCE_CRT = 9 @@ -26,6 +29,7 @@ def generate(args): print(" }") if p >= PRECISION_FORCE_CRT: print("encoding: crt") + print(f"p-error: {P_ERROR}") random_input = np.random.randint(max_value+1, size=n_ct) print("tests:") print(" - inputs:") @@ -62,4 +66,6 @@ CLI.add_argument( type=int, default=[1,2], ) -generate(CLI.parse_args()) + +if __name__ == "__main__": + generate(CLI.parse_args()) diff --git a/compiler/tests/end_to_end_fixture/end_to_end_linalg_leveled_gen.py b/compiler/tests/end_to_end_fixture/end_to_end_linalg_leveled_gen.py index b22fcd664..8cc20d95b 100644 --- a/compiler/tests/end_to_end_fixture/end_to_end_linalg_leveled_gen.py +++ b/compiler/tests/end_to_end_fixture/end_to_end_linalg_leveled_gen.py @@ -2,6 +2,7 @@ import numpy as np PRECISIONS_TO_BENCH = [1, 2, 5, 8, 9, 12, 16, 24, 32, 40, 48, 56] N_CT = [100, 1000, 100000] +P_ERROR = 1.0 - 0.999936657516 def main(): @@ -25,6 +26,7 @@ def main(): " %1 = \"FHELinalg.add_eint_int\"(%arg0, %0): (tensor<{1}x!FHE.eint<{0}>>, tensor<{1}xi{2}>) -> (tensor<{1}x!FHE.eint<{0}>>)".format(p, n_ct, integer_bitwidth)) print(" return %1: tensor<{1}x!FHE.eint<{0}>>".format(p, n_ct)) print(" }") + print(f"p-error: {P_ERROR}") print("tests:") print(" - inputs:") print( @@ -41,6 +43,7 @@ def main(): " %1 = \"FHELinalg.add_eint\"(%arg0, %arg1): (tensor<{1}x!FHE.eint<{0}>>, tensor<{1}x!FHE.eint<{0}>>) -> (tensor<{1}x!FHE.eint<{0}>>)".format(p, n_ct, integer_bitwidth)) print(" return %1: tensor<{1}x!FHE.eint<{0}>>".format(p, n_ct)) print(" }") + print(f"p-error: {P_ERROR}") print("tests:") print(" - inputs:") print( @@ -63,6 +66,7 @@ def main(): print(" return %1: tensor<{1}x!FHE.eint<{0}>>".format( p, n_ct, integer_bitwidth)) print(" }") + print(f"p-error: {P_ERROR}") print("tests:") print(" - inputs:") print( @@ -70,5 +74,5 @@ def main(): print(" shape: [{0}]".format(n_ct)) print("---") - -main() +if __name__ == "__main__": + main() diff --git a/compiler/tests/end_to_end_tests/end_to_end_jit_test.h b/compiler/tests/end_to_end_tests/end_to_end_jit_test.h index d35a47be0..b30dd910f 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_jit_test.h +++ b/compiler/tests/end_to_end_tests/end_to_end_jit_test.h @@ -20,11 +20,12 @@ inline llvm::Expected< internalCheckedJit(llvm::StringRef src, llvm::StringRef func = "main", bool useDefaultFHEConstraints = false, bool dataflowParallelize = false, - bool loopParallelize = false, - bool batchConcreteOps = false) { + bool loopParallelize = false, bool batchConcreteOps = false, + double global_p_error = TEST_ERROR_RATE) { auto options = mlir::concretelang::CompilationOptions(std::string(func.data())); + options.optimizerConfig.global_p_error = global_p_error; if (useDefaultFHEConstraints) { options.v0FHEConstraints = defaultV0Constraints; options.optimizerConfig.strategy_v0 = true; diff --git a/compiler/tests/end_to_end_tests/end_to_end_test.cc b/compiler/tests/end_to_end_tests/end_to_end_test.cc index c3bbc290f..cfe77962b 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_test.cc +++ b/compiler/tests/end_to_end_tests/end_to_end_test.cc @@ -186,6 +186,10 @@ void registerEndToEnd(std::string suiteName, std::string libpath, options.v0FHEConstraints = desc.v0Constraint; } options.optimizerConfig.encoding = desc.encoding; + if (desc.p_error.hasValue()) { + options.optimizerConfig.p_error = *desc.p_error; + options.optimizerConfig.global_p_error = NAN; + } auto i = 0; for (auto test : desc.tests) { auto valueName = std::to_string(i); diff --git a/compiler/tests/end_to_end_tests/end_to_end_test.h b/compiler/tests/end_to_end_tests/end_to_end_test.h index 04d7e3935..190712e7c 100644 --- a/compiler/tests/end_to_end_tests/end_to_end_test.h +++ b/compiler/tests/end_to_end_tests/end_to_end_test.h @@ -12,6 +12,8 @@ static inline uint16_t operator"" _u16(unsigned long long int v) { return v; } static inline uint32_t operator"" _u32(unsigned long long int v) { return v; } static inline uint64_t operator"" _u64(unsigned long long int v) { return v; } +const double TEST_ERROR_RATE = 1.0 - 0.999936657516; + // Evaluates to the number of elements of a statically initialized // array #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))