chore(ci): add dedicatd param-prod run

.github/workflows/aws_tfhe_param_prod_tests.yml

@@ -0,0 +1,280 @@
+# Run a small subset of tests to ensure quick feedback.
+name: aws_tfhe_param_prod_tests
+
+env:
+  CARGO_TERM_COLOR: always
+  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  RUSTFLAGS: "-C target-cpu=native"
+  RUST_BACKTRACE: "full"
+  RUST_MIN_STACK: "8388608"
+  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+  SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+  SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+  SLACKIFY_MARKDOWN: true
+  IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }}
+  PULL_REQUEST_MD_LINK: ""
+  CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }}
+  # Secrets will be available only to zama-ai organization members
+  SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }}
+  EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04"
+
+on:
+  # Allows you to run this workflow manually from the Actions tab as an alternative.
+  workflow_dispatch:
+  pull_request:
+
+permissions:
+  contents: read
+
+# zizmor: ignore[concurrency-limits] concurrency is managed after instance setup to ensure safe provisioning
+
+jobs:
+  should-run:
+    name: aws_tfhe_param_prod_tests/should-run
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read # Needed to check for file change
+    outputs:
+      csprng_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.csprng_any_changed }}
+      zk_pok_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.zk_pok_any_changed }}
+      versionable_test: ${{ env.IS_PULL_REQUEST == 'false' || steps.changed-files.outputs.versionable_any_changed }}
+      core_crypto_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.core_crypto_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      boolean_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.boolean_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      shortint_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.shortint_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      integer_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.integer_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      wasm_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.wasm_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      high_level_api_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.high_level_api_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      user_docs_test: ${{ env.IS_PULL_REQUEST == 'false' ||
+        steps.changed-files.outputs.user_docs_any_changed ||
+        steps.changed-files.outputs.dependencies_any_changed }}
+      any_file_changed: ${{ env.IS_PULL_REQUEST == 'false' || steps.aggregated-changes.outputs.any_changed }}
+    steps:
+      - name: Checkout tfhe-rs
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        with:
+          fetch-depth: 0
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Check for file changes
+        id: changed-files
+        uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0
+        with:
+          files_yaml: |
+            dependencies:
+              - tfhe/Cargo.toml
+              - tfhe-csprng/**
+              - tfhe-fft/**
+              - tfhe-zk-pok/**
+              - utils/tfhe-versionable/**
+              - utils/tfhe-versionable-derive/**
+            csprng:
+              - tfhe-csprng/**
+            zk_pok:
+              - tfhe-zk-pok/**
+            versionable:
+              - utils/tfhe-versionable/**
+              - utils/tfhe-versionable-derive/**
+            core_crypto:
+              - tfhe/src/core_crypto/**
+            boolean:
+              - tfhe/src/core_crypto/**
+              - tfhe/src/boolean/**
+            shortint:
+              - tfhe/src/core_crypto/**
+              - tfhe/src/shortint/**
+            integer:
+              - tfhe/src/core_crypto/**
+              - tfhe/src/shortint/**
+              - tfhe/src/integer/**
+            wasm:
+              - tfhe/src/**
+              - tfhe/js_on_wasm_tests/**
+              - tfhe/web_wasm_parallel_tests/**
+              - '!tfhe/src/c_api/**'
+              - '!tfhe/src/boolean/**'
+            high_level_api:
+              - tfhe/src/**
+              - '!tfhe/src/c_api/**'
+              - '!tfhe/src/boolean/**'
+              - '!tfhe/src/c_api/**'
+              - '!tfhe/src/js_on_wasm_api/**'
+            user_docs:
+              - tfhe/src/**
+              - '!tfhe/src/c_api/**'
+              - 'tfhe/docs/**/**.md'
+              - README.md
+
+      - name: Aggregate file changes
+        id: aggregated-changes
+        if: ( steps.changed-files.outputs.dependencies_any_changed == 'true' ||
+          steps.changed-files.outputs.csprng_any_changed == 'true' ||
+          steps.changed-files.outputs.zk_pok_any_changed == 'true' ||
+          steps.changed-files.outputs.versionable_any_changed == 'true' ||
+          steps.changed-files.outputs.core_crypto_any_changed == 'true' ||
+          steps.changed-files.outputs.boolean_any_changed == 'true' ||
+          steps.changed-files.outputs.shortint_any_changed == 'true' ||
+          steps.changed-files.outputs.integer_any_changed == 'true' ||
+          steps.changed-files.outputs.wasm_any_changed == 'true' ||
+          steps.changed-files.outputs.high_level_api_any_changed == 'true' ||
+          steps.changed-files.outputs.user_docs_any_changed == 'true')
+        run: |
+          echo "any_changed=true" >> "$GITHUB_OUTPUT"
+
+  setup-instance:
+    name: aws_tfhe_param_prod_tests/setup-instance
+    if: github.event_name == 'workflow_dispatch' ||
+      (github.event_name != 'workflow_dispatch' && needs.should-run.outputs.any_file_changed == 'true')
+    needs: should-run
+    runs-on: ubuntu-latest
+    outputs:
+      runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }}
+    steps:
+      - name: Start remote instance
+        id: start-remote-instance
+        if: env.SECRETS_AVAILABLE == 'true'
+        uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2
+        with:
+          mode: start
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          backend: aws
+          profile: cpu-big
+
+      # This instance will be spawned especially for pull-request from forked repository
+      - name: Start GitHub instance
+        id: start-github-instance
+        if: env.SECRETS_AVAILABLE == 'false'
+        run: |
+          echo "runner_group=${EXTERNAL_CONTRIBUTION_RUNNER}" >> "$GITHUB_OUTPUT"
+
+  param-prod-tests:
+    name: Fast CPU tests
+    needs: [ should-run, setup-instance ]
+    concurrency:
+      group: ${{ github.workflow_ref }}
+      cancel-in-progress: true
+    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
+    steps:
+      - name: Checkout tfhe-rs
+        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
+        with:
+          persist-credentials: 'false'
+          token: ${{ env.CHECKOUT_TOKEN }}
+
+      - name: Install latest stable
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases
+        with:
+          toolchain: stable
+
+      - name: Run user docs tests
+        if: needs.should-run.outputs.user_docs_test == 'true'
+        run: |
+          make test_user_doc
+
+      - name: Get Node version
+        run: |
+          echo "NODE_VERSION=$(make node_version)" >> "${GITHUB_ENV}"
+
+      - name: Node cache restoration
+        id: node-cache
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0
+        with:
+          path: |
+            ~/.nvm
+            ~/.npm
+          key: node-${{ env.NODE_VERSION }}
+
+      - name: Install Node
+        if: steps.node-cache.outputs.cache-hit != 'true'
+        run: |
+          make install_node
+
+      - name: Node cache save
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0
+        if: steps.node-cache.outputs.cache-hit != 'true'
+        with:
+          path: |
+            ~/.nvm
+            ~/.npm
+          key: node-${{ env.NODE_VERSION }}
+
+      - name: Run js on wasm API tests
+        if: needs.should-run.outputs.wasm_test == 'true'
+        run: |
+          make test_nodejs_wasm_api_ci
+
+      - name: Gen Keys if required
+        if: needs.should-run.outputs.shortint_test == 'true' ||
+          needs.should-run.outputs.integer_test == 'true'
+        run: |
+          make gen_key_cache
+
+      - name: Run shortint tests
+        if: needs.should-run.outputs.shortint_test == 'true'
+        run: |
+          BIG_TESTS_INSTANCE=TRUE FAST_TESTS=FALSE make test_shortint_ci
+
+      - name: Run integer tests
+        if: needs.should-run.outputs.integer_test == 'true'
+        run: |
+          BIG_TESTS_INSTANCE=TRUE FAST_TESTS=FALSE make test_param_prod_integer_ci 
+
+      - name: Run high-level API tests
+        if: needs.should-run.outputs.high_level_api_test == 'true'
+        run: |
+          make test_high_level_api
+
+      - name: Set pull-request URL
+        if: ${{ failure() && github.event_name == 'pull_request' }}
+        run: |
+          echo "PULL_REQUEST_MD_LINK=[pull-request](${PR_BASE_URL}${PR_NUMBER}), "  >> "${GITHUB_ENV}"
+        env:
+          PR_BASE_URL: ${{ vars.PR_BASE_URL }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+
+      - name: Slack Notification
+        if: ${{ failure() && env.SECRETS_AVAILABLE == 'true' }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "Fast AWS tests finished with status: ${{ job.status }}. (${{ env.PULL_REQUEST_MD_LINK }}[action run](${{ env.ACTION_RUN_URL }}))"
+
+  teardown-instance:
+    name: aws_tfhe_param_prod_tests/teardown-instance
+    if: ${{ always() && needs.setup-instance.result == 'success' }}
+    needs: [ setup-instance, param-prod-tests ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Stop remote instance
+        id: stop-instance
+        if: env.SECRETS_AVAILABLE == 'true'
+        uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2
+        with:
+          mode: stop
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          label: ${{ needs.setup-instance.outputs.runner-name }}
+
+      - name: Slack Notification
+        if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }}
+        uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "Instance teardown (param-prod-tests) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

Makefile

@@ -934,6 +934,15 @@ test_integer_ci: install_cargo_nextest
 		--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \
 		--tfhe-package "tfhe"
 
+.PHONY: test_param_prod_integer_ci # Run the tests for integer ci
+test_param_prod_integer_ci: install_cargo_nextest
+	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
+	FAST_TESTS="$(FAST_TESTS)" \
+	NIGHTLY_TESTS="$(NIGHTLY_TESTS)" \
+		./scripts/integer-tests.sh \
+		--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \
+		--run-prod-only --tfhe-package "tfhe"
+
 .PHONY: test_unsigned_integer_ci # Run the tests for unsigned integer ci
 test_unsigned_integer_ci: install_cargo_nextest
 	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \

scripts/integer-tests.sh

@@ -6,10 +6,12 @@ function usage() {
     echo "$0: integer test runner"
     echo
     echo "--help                    Print this message"
+    echo "--no-run                  Does not run the tests, but prints which tests would be ran (except doctests)"
     echo "--rust-toolchain          The toolchain to run the tests with default: stable"
     echo "--multi-bit               Run multi-bit tests only: default off"
     echo "--unsigned-only           Run only unsigned integer tests, by default both signed and unsigned tests are run"
     echo "--signed-only             Run only signed integer tests, by default both signed and unsigned tests are run"
+    echo "--run-prod-only           Run only the tests using the prod parameters"
     echo "--nightly-tests           Run integer tests configured for nightly runs (3_3 params)"
     echo "--fast-tests              Run integer set but skip a subset of longer tests"
     echo "--long-tests              Run only long run integer tests"
@@ -33,6 +35,8 @@ backend="cpu"
 gpu_feature=""
 avx512_feature=""
 tfhe_package="tfhe"
+prod_param_argument=
+no_run=false
 
 while [ -n "$1" ]
 do
@@ -42,6 +46,14 @@ do
             exit 0
             ;;
 
+        "--no-run" )
+            no_run=true
+            ;;
+
+        "--run-prod-only" )
+            prod_param_argument="--run-prod-only"
+            ;;
+
         "--rust-toolchain" )
             shift
             RUST_TOOLCHAIN="$1"
@@ -150,7 +162,7 @@ if [[ "${backend}" == "gpu" ]]; then
     fi
 fi
 
-filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument:+$fast_tests_argument} ${long_tests_argument:+$long_tests_argument} ${nightly_tests_argument:+$nightly_tests_argument} ${no_big_params_argument_gpu:+$no_big_params_argument_gpu} ${multi_bit_argument:+$multi_bit_argument} ${sign_argument:+$sign_argument} ${no_big_params_argument:+$no_big_params_argument})
+filter_expression=$(/usr/bin/python3 scripts/test_filtering.py --layer integer --backend "${backend}" ${fast_tests_argument:+$fast_tests_argument} ${long_tests_argument:+$long_tests_argument} ${nightly_tests_argument:+$nightly_tests_argument} ${no_big_params_argument_gpu:+$no_big_params_argument_gpu} ${multi_bit_argument:+$multi_bit_argument} ${sign_argument:+$sign_argument} ${no_big_params_argument:+$no_big_params_argument} ${prod_param_argument:+$prod_param_argument})
 
 if [[ "${FAST_TESTS}" == "TRUE" ]]; then
     echo "Running 'fast' test set"
@@ -168,24 +180,39 @@ fi
 
 echo "${filter_expression}"
 
-cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest run \
-    --tests \
-    --cargo-profile "${cargo_profile}" \
-    --package "${tfhe_package}" \
-    --profile ci \
-    --no-default-features \
-    --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \
-    --test-threads "${test_threads}" \
-    -E "$filter_expression"
+if $no_run then; then
+    # This is very close to the run command, but the `--profile ci`
+    # is not 'supported'
+    cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest list \
+        --tests \
+        --cargo-profile "${cargo_profile}" \
+        --package "${tfhe_package}" \
+        --no-default-features \
+        --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \
+        -E "$filter_expression"
+else
+    cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} nextest run \
+        --tests \
+        --cargo-profile "${cargo_profile}" \
+        --package "${tfhe_package}" \
+        --profile ci \
+        --no-default-features \
+        --features=integer,internal-keycache,zk-pok,experimental,"${avx512_feature}","${gpu_feature}" \
+        --test-threads "${test_threads}" \
+        -E "$filter_expression"
 
-if [[ -z ${multi_bit_argument} && -z ${long_tests_argument} ]]; then
-    cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} test \
+    # Unfortunately, we cannot skip running doctest with `--no-run`
+    if [[ -z ${multi_bit_argument} && -z ${long_tests_argument} ]]; then
+      cargo ${RUST_TOOLCHAIN:+"$RUST_TOOLCHAIN"} test \
         --profile "${cargo_profile}" \
         --package "${tfhe_package}" \
         --no-default-features \
         --features=integer,internal-keycache,experimental,"${avx512_feature}","${gpu_feature}" \
         --doc \
         -- --test-threads="${doctest_threads}" integer::"${gpu_feature}"
+    fi
 fi
 
+
+
 echo "Test ran in $SECONDS seconds"

scripts/test_filtering.py

@@ -121,18 +121,17 @@ def filter_integer_tests(input_args):
         ("_multi_bit", "_group_[0-9]") if input_args.multi_bit else ("", "")
     )
     backend_filter = ""
-    if input_args.run_prod_only:
-        filter_expression = [
-            "test(/^integer::.*_param_prod/)",
-        ]
-    elif not input_args.long_tests:
+    if not input_args.long_tests:
         if input_args.backend == "gpu":
             backend_filter = "gpu::"
             if multi_bit_filter:
                 # For now, GPU only has specific parameters set for multi-bit
                 multi_bit_filter = "_gpu_multi_bit"
 
-        filter_expression = [f"test(/^integer::{backend_filter}.*/)"]
+        if input_args.run_prod_only:
+            filter_expression = [f"test(/^integer::{backend_filter}.*_param_prod/)"]
+        else:
+            filter_expression = [f"test(/^integer::{backend_filter}.*/)"]
 
         if input_args.multi_bit:
             filter_expression.append("test(~_multi_bit)")
@@ -175,6 +174,9 @@ def filter_integer_tests(input_args):
         for pattern in excluded_tests:
             filter_expression.append(f"not test({pattern})")
 
+        if not input_args.run_prod_only:
+            filter_expression.append("not test(/.*_param_prod_.*/)")
+
     else:
         if input_args.backend == "gpu":
             filter_expression = [
@@ -184,11 +186,10 @@ def filter_integer_tests(input_args):
             filter_expression = [
                 "test(/^integer::server_key::radix_parallel::tests_long_run.*/)"
             ]
-
     # Do not run noise check tests by default as they can be very slow
     # they will be run e.g. nightly or on demand
     filter_expression.append(
-        f"not test(/^integer::gpu::server_key::radix::tests_noise_distribution::.*::test_gpu_noise_check.*/)"
+        "not test(/^integer::gpu::server_key::radix::tests_noise_distribution::.*::test_gpu_noise_check.*/)"
     )
     return " and ".join(filter_expression)