diff --git a/Makefile b/Makefile index 95b191fd1..cd901105f 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ GEN_KEY_CACHE_COVERAGE_ONLY?=FALSE PARSE_INTEGER_BENCH_CSV_FILE?=tfhe_rs_integer_benches.csv FAST_TESTS?=FALSE FAST_BENCH?=FALSE +NIGHTLY_TESTS?=FALSE BENCH_OP_FLAVOR?=DEFAULT NODE_VERSION=22 FORWARD_COMPAT?=OFF @@ -481,6 +482,7 @@ test_integer_gpu: install_rs_build_toolchain test_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --backend "gpu" \ --tfhe-package "$(TFHE_SPEC)" @@ -489,6 +491,7 @@ test_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest test_unsigned_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --backend "gpu" \ --unsigned-only --tfhe-package "$(TFHE_SPEC)" @@ -497,6 +500,7 @@ test_unsigned_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest test_signed_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --backend "gpu" \ --signed-only --tfhe-package "$(TFHE_SPEC)" @@ -505,6 +509,7 @@ test_signed_integer_gpu_ci: install_rs_check_toolchain install_cargo_nextest test_integer_multi_bit_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --backend "gpu" \ --tfhe-package "$(TFHE_SPEC)" @@ -513,6 +518,7 @@ test_integer_multi_bit_gpu_ci: install_rs_check_toolchain install_cargo_nextest test_unsigned_integer_multi_bit_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --backend "gpu" \ --unsigned-only --tfhe-package "$(TFHE_SPEC)" @@ -521,6 +527,7 @@ test_unsigned_integer_multi_bit_gpu_ci: install_rs_check_toolchain install_cargo test_signed_integer_multi_bit_gpu_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --backend "gpu" \ --signed-only --tfhe-package "$(TFHE_SPEC)" @@ -587,6 +594,7 @@ test_shortint_cov: install_rs_check_toolchain install_tarpaulin test_integer_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \ --tfhe-package "$(TFHE_SPEC)" @@ -595,6 +603,7 @@ test_integer_ci: install_rs_check_toolchain install_cargo_nextest test_unsigned_integer_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \ --unsigned-only --tfhe-package "$(TFHE_SPEC)" @@ -603,6 +612,7 @@ test_unsigned_integer_ci: install_rs_check_toolchain install_cargo_nextest test_signed_integer_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \ --signed-only --tfhe-package "$(TFHE_SPEC)" @@ -611,6 +621,7 @@ test_signed_integer_ci: install_rs_check_toolchain install_cargo_nextest test_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \ --tfhe-package "$(TFHE_SPEC)" @@ -619,6 +630,7 @@ test_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest test_unsigned_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \ --unsigned-only --tfhe-package "$(TFHE_SPEC)" @@ -627,6 +639,7 @@ test_unsigned_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nex test_signed_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ ./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \ --cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \ --signed-only --tfhe-package "$(TFHE_SPEC)" diff --git a/scripts/integer-tests.sh b/scripts/integer-tests.sh index 5477e6014..98258edae 100755 --- a/scripts/integer-tests.sh +++ b/scripts/integer-tests.sh @@ -21,6 +21,7 @@ RUST_TOOLCHAIN="+stable" multi_bit_argument= sign_argument= fast_tests_argument= +nightly_tests_argument= no_big_params_argument= cargo_profile="release" backend="cpu" @@ -90,6 +91,10 @@ if [[ "${FAST_TESTS}" == TRUE ]]; then fast_tests_argument=--fast-tests fi +if [[ "${NIGHTLY_TESTS}" == TRUE ]]; then + nightly_tests_argument=--nightly-tests +fi + if [[ "${NO_BIG_PARAMS}" == TRUE ]]; then no_big_params_argument=--no-big-params fi @@ -127,14 +132,18 @@ if [[ "${backend}" == "gpu" ]]; then test_threads=6 fi -filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument} ${multi_bit_argument} ${sign_argument} ${no_big_params_argument}) +filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument} ${nightly_tests_argument} ${multi_bit_argument} ${sign_argument} ${no_big_params_argument}) if [[ "${FAST_TESTS}" == "TRUE" ]]; then - echo "Running 'fast' test set'" + echo "Running 'fast' test set" else echo "Running 'slow' test set" fi +if [[ "${NIGHTLY_TESTS}" == "TRUE" ]]; then + echo "Running 'nightly' test set" +fi + cargo "${RUST_TOOLCHAIN}" nextest run \ --tests \ --cargo-profile "${cargo_profile}" \ diff --git a/scripts/test_filtering.py b/scripts/test_filtering.py index 43d741076..b9fdbce04 100644 --- a/scripts/test_filtering.py +++ b/scripts/test_filtering.py @@ -2,6 +2,7 @@ Script that generates a cargo-nextest filter as an output. The string result can be directly injected into a nextest command. """ + import argparse parser = argparse.ArgumentParser(allow_abbrev=False) @@ -25,6 +26,12 @@ parser.add_argument( action="store_true", help="Run only a small subset of test suite", ) +parser.add_argument( + "--nightly-tests", + dest="nightly_tests", + action="store_true", + help="Run only a subset of test suite", +) parser.add_argument( "--big-instance", dest="big_instance", @@ -87,6 +94,7 @@ EXCLUDED_BIG_PARAMETERS = [ "/.*_param_message_4_carry_4_ks_pbs$/", ] + def filter_integer_tests(input_args): multi_bit_filter = "_multi_bit" if input_args.multi_bit else "" backend_filter = "" @@ -109,11 +117,20 @@ def filter_integer_tests(input_args): for pattern in EXCLUDED_BIG_PARAMETERS: filter_expression.append(f"not test({pattern})") - if input_args.fast_tests: - # Test only fast default operations with only one set of parameters - param_group = "_group_2" if input_args.multi_bit else "" + if input_args.fast_tests and input_args.nightly_tests: filter_expression.append( - f"test(/.*_default_.*?_param{multi_bit_filter}_message_2_carry_2{param_group}_.*/)" + f"test(/.*_default_.*?_param{multi_bit_filter}_message_[2-3]_carry_[2-3]_.*/)" + ) + elif input_args.fast_tests: + # Test only fast default operations with only one set of parameters + filter_expression.append( + f"test(/.*_default_.*?_param{multi_bit_filter}_message_2_carry_2_.*/)" + ) + elif input_args.nightly_tests: + # Test only fast default operations with only one set of parameters + # This subset would run slower than fast_tests hence the use of nightly_tests + filter_expression.append( + f"test(/.*_default_.*?_param{multi_bit_filter}_message_3_carry_3_.*/)" ) excluded_tests = (