diff --git a/.github/workflows/aws_tfhe_param_prod_tests.yml b/.github/workflows/aws_tfhe_param_prod_tests.yml new file mode 100644 index 000000000..ead5e98ab --- /dev/null +++ b/.github/workflows/aws_tfhe_param_prod_tests.yml @@ -0,0 +1,280 @@ +# Run a small subset of tests to ensure quick feedback. +name: aws_tfhe_param_prod_tests + +env: + CARGO_TERM_COLOR: always + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUSTFLAGS: "-C target-cpu=native" + RUST_BACKTRACE: "full" + RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACKIFY_MARKDOWN: true + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + PULL_REQUEST_MD_LINK: "" + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04" + +on: + # Allows you to run this workflow manually from the Actions tab as an alternative. + workflow_dispatch: + pull_request: + +permissions: + contents: read + +# zizmor: ignore[concurrency-limits] concurrency is managed after instance setup to ensure safe provisioning + +jobs: + should-run: + name: aws_tfhe_param_prod_tests/should-run + runs-on: ubuntu-latest + permissions: + pull-requests: read # Needed to check for file change + outputs: + csprng_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.csprng_any_changed }} + zk_pok_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.zk_pok_any_changed }} + versionable_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.versionable_any_changed }} + core_crypto_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.core_crypto_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + boolean_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.boolean_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + shortint_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.shortint_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + integer_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.integer_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + wasm_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.wasm_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + high_level_api_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.high_level_api_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + user_docs_test: ${{ env.IS_PULL_REQUEST == 'false' || + steps.changed-files.outputs.user_docs_any_changed || + steps.changed-files.outputs.dependencies_any_changed }} + any_file_changed: ${{ env.IS_PULL_REQUEST == 'false' || steps.aggregated-changes.outputs.any_changed }} + steps: + - name: Checkout tfhe-rs + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 + with: + fetch-depth: 0 + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Check for file changes + id: changed-files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + with: + files_yaml: | + dependencies: + - tfhe/Cargo.toml + - tfhe-csprng/** + - tfhe-fft/** + - tfhe-zk-pok/** + - utils/tfhe-versionable/** + - utils/tfhe-versionable-derive/** + csprng: + - tfhe-csprng/** + zk_pok: + - tfhe-zk-pok/** + versionable: + - utils/tfhe-versionable/** + - utils/tfhe-versionable-derive/** + core_crypto: + - tfhe/src/core_crypto/** + boolean: + - tfhe/src/core_crypto/** + - tfhe/src/boolean/** + shortint: + - tfhe/src/core_crypto/** + - tfhe/src/shortint/** + integer: + - tfhe/src/core_crypto/** + - tfhe/src/shortint/** + - tfhe/src/integer/** + wasm: + - tfhe/src/** + - tfhe/js_on_wasm_tests/** + - tfhe/web_wasm_parallel_tests/** + - '!tfhe/src/c_api/**' + - '!tfhe/src/boolean/**' + high_level_api: + - tfhe/src/** + - '!tfhe/src/c_api/**' + - '!tfhe/src/boolean/**' + - '!tfhe/src/c_api/**' + - '!tfhe/src/js_on_wasm_api/**' + user_docs: + - tfhe/src/** + - '!tfhe/src/c_api/**' + - 'tfhe/docs/**/**.md' + - README.md + + - name: Aggregate file changes + id: aggregated-changes + if: ( steps.changed-files.outputs.dependencies_any_changed == 'true' || + steps.changed-files.outputs.csprng_any_changed == 'true' || + steps.changed-files.outputs.zk_pok_any_changed == 'true' || + steps.changed-files.outputs.versionable_any_changed == 'true' || + steps.changed-files.outputs.core_crypto_any_changed == 'true' || + steps.changed-files.outputs.boolean_any_changed == 'true' || + steps.changed-files.outputs.shortint_any_changed == 'true' || + steps.changed-files.outputs.integer_any_changed == 'true' || + steps.changed-files.outputs.wasm_any_changed == 'true' || + steps.changed-files.outputs.high_level_api_any_changed == 'true' || + steps.changed-files.outputs.user_docs_any_changed == 'true') + run: | + echo "any_changed=true" >> "$GITHUB_OUTPUT" + + setup-instance: + name: aws_tfhe_param_prod_tests/setup-instance + if: github.event_name == 'workflow_dispatch' || + (github.event_name != 'workflow_dispatch' && needs.should-run.outputs.any_file_changed == 'true') + needs: should-run + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} + steps: + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' + uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: cpu-big + + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${EXTERNAL_CONTRIBUTION_RUNNER}" >> "$GITHUB_OUTPUT" + + param-prod-tests: + name: Fast CPU tests + needs: [ should-run, setup-instance ] + concurrency: + group: ${{ github.workflow_ref }} + cancel-in-progress: true + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + steps: + - name: Checkout tfhe-rs + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 + with: + persist-credentials: 'false' + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install latest stable + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases + with: + toolchain: stable + + - name: Run user docs tests + if: needs.should-run.outputs.user_docs_test == 'true' + run: | + make test_user_doc + + - name: Get Node version + run: | + echo "NODE_VERSION=$(make node_version)" >> "${GITHUB_ENV}" + + - name: Node cache restoration + id: node-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0 + with: + path: | + ~/.nvm + ~/.npm + key: node-${{ env.NODE_VERSION }} + + - name: Install Node + if: steps.node-cache.outputs.cache-hit != 'true' + run: | + make install_node + + - name: Node cache save + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0 + if: steps.node-cache.outputs.cache-hit != 'true' + with: + path: | + ~/.nvm + ~/.npm + key: node-${{ env.NODE_VERSION }} + + - name: Run js on wasm API tests + if: needs.should-run.outputs.wasm_test == 'true' + run: | + make test_nodejs_wasm_api_ci + + - name: Gen Keys if required + if: needs.should-run.outputs.shortint_test == 'true' || + needs.should-run.outputs.integer_test == 'true' + run: | + make gen_key_cache + + - name: Run shortint tests + if: needs.should-run.outputs.shortint_test == 'true' + run: | + BIG_TESTS_INSTANCE=TRUE FAST_TESTS=FALSE make test_shortint_ci + + - name: Run integer tests + if: needs.should-run.outputs.integer_test == 'true' + run: | + BIG_TESTS_INSTANCE=TRUE FAST_TESTS=FALSE make test_param_prod_integer_ci + + - name: Run high-level API tests + if: needs.should-run.outputs.high_level_api_test == 'true' + run: | + make test_high_level_api + + - name: Set pull-request URL + if: ${{ failure() && github.event_name == 'pull_request' }} + run: | + echo "PULL_REQUEST_MD_LINK=[pull-request](${PR_BASE_URL}${PR_NUMBER}), " >> "${GITHUB_ENV}" + env: + PR_BASE_URL: ${{ vars.PR_BASE_URL }} + PR_NUMBER: ${{ github.event.pull_request.number }} + + - name: Slack Notification + if: ${{ failure() && env.SECRETS_AVAILABLE == 'true' }} + continue-on-error: true + uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Fast AWS tests finished with status: ${{ job.status }}. (${{ env.PULL_REQUEST_MD_LINK }}[action run](${{ env.ACTION_RUN_URL }}))" + + teardown-instance: + name: aws_tfhe_param_prod_tests/teardown-instance + if: ${{ always() && needs.setup-instance.result == 'success' }} + needs: [ setup-instance, param-prod-tests ] + runs-on: ubuntu-latest + steps: + - name: Stop remote instance + id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' + uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} + uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (param-prod-tests) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/Makefile b/Makefile index bfb84e2fd..304d4de37 100644 --- a/Makefile +++ b/Makefile @@ -934,6 +934,15 @@ test_integer_ci: install_cargo_nextest --cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \ --tfhe-package "tfhe" +.PHONY: test_param_prod_integer_ci # Run the tests for integer ci +test_param_prod_integer_ci: install_cargo_nextest + BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ + FAST_TESTS="$(FAST_TESTS)" \ + NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \ + ./scripts/integer-tests.sh \ + --cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \ + --run-prod-only --tfhe-package "tfhe" + .PHONY: test_unsigned_integer_ci # Run the tests for unsigned integer ci test_unsigned_integer_ci: install_cargo_nextest BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \ diff --git a/scripts/integer-tests.sh b/scripts/integer-tests.sh index 2be5c0b6d..c9c837a86 100755 --- a/scripts/integer-tests.sh +++ b/scripts/integer-tests.sh @@ -6,10 +6,12 @@ function usage() { echo "$0: integer test runner" echo echo "--help Print this message" + echo "--no-run Does not run the tests, but prints which tests would be ran (except doctests)" echo "--rust-toolchain The toolchain to run the tests with default: stable" echo "--multi-bit Run multi-bit tests only: default off" echo "--unsigned-only Run only unsigned integer tests, by default both signed and unsigned tests are run" echo "--signed-only Run only signed integer tests, by default both signed and unsigned tests are run" + echo "--run-prod-only Run only the tests using the prod parameters" echo "--nightly-tests Run integer tests configured for nightly runs (3_3 params)" echo "--fast-tests Run integer set but skip a subset of longer tests" echo "--long-tests Run only long run integer tests" @@ -33,6 +35,8 @@ backend="cpu" gpu_feature="" avx512_feature="" tfhe_package="tfhe" +prod_param_argument= +no_run=false while [ -n "$1" ] do @@ -42,6 +46,14 @@ do exit 0 ;; + "--no-run" ) + no_run=true + ;; + + "--run-prod-only" ) + prod_param_argument="--run-prod-only" + ;; + "--rust-toolchain" ) shift RUST_TOOLCHAIN="$1" @@ -150,7 +162,7 @@ if [[ "${backend}" == "gpu" ]]; then fi fi -filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument:+$fast_tests_argument} ${long_tests_argument:+$long_tests_argument} ${nightly_tests_argument:+$nightly_tests_argument} ${no_big_params_argument_gpu:+$no_big_params_argument_gpu} ${multi_bit_argument:+$multi_bit_argument} ${sign_argument:+$sign_argument} ${no_big_params_argument:+$no_big_params_argument}) +filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument:+$fast_tests_argument} ${long_tests_argument:+$long_tests_argument} ${nightly_tests_argument:+$nightly_tests_argument} ${no_big_params_argument_gpu:+$no_big_params_argument_gpu} ${multi_bit_argument:+$multi_bit_argument} ${sign_argument:+$sign_argument} ${no_big_params_argument:+$no_big_params_argument} ${prod_param_argument:+$prod_param_argument}) if [[ "${FAST_TESTS}" == "TRUE" ]]; then echo "Running 'fast' test set" @@ -168,24 +180,39 @@ fi echo "${filter_expression}" -cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest run \ - --tests \ - --cargo-profile "${cargo_profile}" \ - --package "${tfhe_package}" \ - --profile ci \ - --no-default-features \ - --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \ - --test-threads "${test_threads}" \ - -E "$filter_expression" +if $no_run then; then + # This is very close to the run command, but the `--profile ci` + # is not 'supported' + cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest list \ + --tests \ + --cargo-profile "${cargo_profile}" \ + --package "${tfhe_package}" \ + --no-default-features \ + --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \ + -E "$filter_expression" +else + cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest run \ + --tests \ + --cargo-profile "${cargo_profile}" \ + --package "${tfhe_package}" \ + --profile ci \ + --no-default-features \ + --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \ + --test-threads "${test_threads}" \ + -E "$filter_expression" -if [[ -z ${multi_bit_argument} && -z ${long_tests_argument} ]]; then - cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} test \ + # Unfortunately, we cannot skip running doctest with `--no-run` + if [[ -z ${multi_bit_argument} && -z ${long_tests_argument} ]]; then + cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} test \ --profile "${cargo_profile}" \ --package "${tfhe_package}" \ --no-default-features \ --features=integer,internal-keycache,experimental,"${avx512_feature}","${gpu_feature}" \ --doc \ -- --test-threads="${doctest_threads}" integer::"${gpu_feature}" + fi fi + + echo "Test ran in $SECONDS seconds" diff --git a/scripts/test_filtering.py b/scripts/test_filtering.py index c189c3d60..3493ac5e0 100644 --- a/scripts/test_filtering.py +++ b/scripts/test_filtering.py @@ -121,18 +121,17 @@ def filter_integer_tests(input_args): ("_multi_bit", "_group_[0-9]") if input_args.multi_bit else ("", "") ) backend_filter = "" - if input_args.run_prod_only: - filter_expression = [ - "test(/^integer::.*_param_prod/)", - ] - elif not input_args.long_tests: + if not input_args.long_tests: if input_args.backend == "gpu": backend_filter = "gpu::" if multi_bit_filter: # For now, GPU only has specific parameters set for multi-bit multi_bit_filter = "_gpu_multi_bit" - filter_expression = [f"test(/^integer::{backend_filter}.*/)"] + if input_args.run_prod_only: + filter_expression = [f"test(/^integer::{backend_filter}.*_param_prod/)"] + else: + filter_expression = [f"test(/^integer::{backend_filter}.*/)"] if input_args.multi_bit: filter_expression.append("test(~_multi_bit)") @@ -175,6 +174,9 @@ def filter_integer_tests(input_args): for pattern in excluded_tests: filter_expression.append(f"not test({pattern})") + if not input_args.run_prod_only: + filter_expression.append("not test(/.*_param_prod_.*/)") + else: if input_args.backend == "gpu": filter_expression = [ @@ -184,11 +186,10 @@ def filter_integer_tests(input_args): filter_expression = [ "test(/^integer::server_key::radix_parallel::tests_long_run.*/)" ] - # Do not run noise check tests by default as they can be very slow # they will be run e.g. nightly or on demand filter_expression.append( - f"not test(/^integer::gpu::server_key::radix::tests_noise_distribution::.*::test_gpu_noise_check.*/)" + "not test(/^integer::gpu::server_key::radix::tests_noise_distribution::.*::test_gpu_noise_check.*/)" ) return " and ".join(filter_expression)