From c3017341bd09cd0eb5d20d9cf001c8fdbdcea96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Wed, 29 Oct 2025 12:26:34 +0100 Subject: [PATCH] chore(ci): refactor cpu benchmarks workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following the same pattern as GPU benchmarks, CPU benchmarks rely on a common workflow. Weekly benchmarks are all gathered in one place. Also, all the manual launches via workflow_dispatch event are now done in one place. That way, one doesn't have to browse the workflow tree to find the right CPU benchmark to trigger. Signed-off-by: David Testé --- .github/workflows/benchmark_boolean.yml | 156 ----------- .github/workflows/benchmark_core_crypto.yml | 234 ----------------- .github/workflows/benchmark_cpu.yml | 81 ++++++ ...k_integer.yml => benchmark_cpu_common.yml} | 208 +++++++++------ .github/workflows/benchmark_cpu_weekly.yml | 220 ++++++++++++++++ .github/workflows/benchmark_dex.yml | 170 ------------ .github/workflows/benchmark_erc20.yml | 153 ----------- .github/workflows/benchmark_shortint.yml | 179 ------------- .../workflows/benchmark_signed_integer.yml | 227 ---------------- .github/workflows/benchmark_tfhe_zk_pok.yml | 198 -------------- .github/workflows/benchmark_zk_pke.yml | 248 ------------------ Makefile | 4 +- 12 files changed, 425 insertions(+), 1653 deletions(-) delete mode 100644 .github/workflows/benchmark_boolean.yml delete mode 100644 .github/workflows/benchmark_core_crypto.yml create mode 100644 .github/workflows/benchmark_cpu.yml rename .github/workflows/{benchmark_integer.yml => benchmark_cpu_common.yml} (52%) create mode 100644 .github/workflows/benchmark_cpu_weekly.yml delete mode 100644 .github/workflows/benchmark_dex.yml delete mode 100644 .github/workflows/benchmark_erc20.yml delete mode 100644 .github/workflows/benchmark_shortint.yml delete mode 100644 .github/workflows/benchmark_signed_integer.yml delete mode 100644 .github/workflows/benchmark_tfhe_zk_pok.yml delete mode 100644 .github/workflows/benchmark_zk_pke.yml diff --git a/.github/workflows/benchmark_boolean.yml b/.github/workflows/benchmark_boolean.yml deleted file mode 100644 index 6e955f8b2..000000000 --- a/.github/workflows/benchmark_boolean.yml +++ /dev/null @@ -1,156 +0,0 @@ -# Run boolean benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_boolean - -on: - workflow_dispatch: - schedule: - # Weekly benchmarks will be triggered each Saturday at 1a.m. - - cron: '0 1 * * 6' - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - -permissions: {} - -jobs: - setup-instance: - name: benchmark_boolean/setup-instance - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - boolean-benchmarks: - name: benchmark_boolean/boolean-benchmarks - needs: setup-instance - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Run benchmarks with AVX512 - run: | - make bench_boolean - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 - env: - REF_NAME: ${{ github.ref_name }} - - - name: Measure key sizes - run: | - make measure_boolean_key_sizes - - - name: Parse key sizes results - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/boolean_key_sizes.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_boolean - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Boolean benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_boolean/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, boolean-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (boolean-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_core_crypto.yml b/.github/workflows/benchmark_core_crypto.yml deleted file mode 100644 index 99c99bc9d..000000000 --- a/.github/workflows/benchmark_core_crypto.yml +++ /dev/null @@ -1,234 +0,0 @@ -# Run core crypto benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_core_crypto - -on: - workflow_dispatch: - inputs: - param_type: - description: "Parameters type" - type: choice - default: classical - options: - - classical - - multi_bit - - classical + multi_bit - - classical_documentation - - multi_bit_documentation - - classical_documentation + multi_bit_documentation - bench_type: - description: "Benchmarks type" - type: choice - default: latency - options: - - latency - - throughput - - both - - schedule: - # Weekly benchmarks will be triggered each Saturday at 5a.m. - - cron: '0 5 * * 6' - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - -permissions: {} - -jobs: - prepare-matrix: - name: benchmark_core_crypto/prepare-matrix - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - param_type: ${{ steps.set_param_type.outputs.param_type }} - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} - steps: - - name: Set parameters types - if: github.event_name == 'workflow_dispatch' - run: | - if [[ "${INPUTS_PARAM_TYPE}" == "classical + multi_bit" ]]; then - echo "PARAM_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}" - elif [[ "${INPUTS_PARAM_TYPE}" == "classical_documentation + multi_bit_documentation" ]]; then - echo "PARAM_TYPE=[\"classical_documentation\", \"multi_bit_documentation\"]" >> "${GITHUB_ENV}" - else - echo "PARAM_TYPE=[\"${INPUTS_PARAM_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_PARAM_TYPE: ${{ inputs.param_type }} - - - name: Default parameters type - if: github.event_name != 'workflow_dispatch' - run: | - echo "PARAM_TYPE=[\"classical\"]" >> "${GITHUB_ENV}" - - - name: Set benchmark types - if: github.event_name == 'workflow_dispatch' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} - - - name: Default benchmark type - if: github.event_name != 'workflow_dispatch' - run: | - echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" - - - name: Set parameters types output - id: set_param_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "param_type=${{ toJSON(env.PARAM_TYPE) }}" >> "${GITHUB_OUTPUT}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" - - setup-instance: - name: benchmark_core_crypto/setup-instance - needs: prepare-matrix - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - core-crypto-benchmarks: - name: benchmark_core_crypto/core-crypto-benchmarks - needs: [ prepare-matrix, setup-instance ] - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - timeout-minutes: 1440 # 24 hours - strategy: - max-parallel: 1 - matrix: - param_type: ${{ fromJSON(needs.prepare-matrix.outputs.param_type) }} - bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Run benchmarks with AVX512 - run: | - make bench_ks_pbs - make bench_pbs - make bench_pbs128 - make bench_ks - env: - BENCH_PARAM_TYPE: ${{ matrix.param_type }} - BENCH_TYPE: ${{ matrix.bench_type }} - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --name-suffix avx512 \ - --walk-subdirs - env: - REF_NAME: ${{ github.ref_name }} - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_core_crypto_${{ matrix.param_type }}_pbs - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "PBS benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_core_crypto/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, core-crypto-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (core-crypto-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_cpu.yml b/.github/workflows/benchmark_cpu.yml new file mode 100644 index 000000000..c0c2af966 --- /dev/null +++ b/.github/workflows/benchmark_cpu.yml @@ -0,0 +1,81 @@ +# Run benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: benchmark_cpu + +on: + workflow_dispatch: + inputs: + command: + description: "Benchmark command to run" + type: choice + options: + - integer + - signed_integer + - integer_compression + - integer_zk + - shortint + - shortint_oprf + - hlapi + - hlapi_erc20 + - hlapi_dex + - hlapi_noise_squash + - tfhe_zk_pok + - boolean + - pbs + - pbs128 + - ks + - ks_pbs + op_flavor: + description: "Operations set to run" + type: choice + default: default + options: + - default + - fast_default + - smart + - unchecked + - misc + all_precisions: + description: "Run all precisions" + type: boolean + default: false + bench_type: + description: "Benchmarks type" + type: choice + default: latency + options: + - latency + - throughput + - both + params_type: + description: "Parameters type" + type: choice + default: classical + options: + - classical + - multi_bit + - classical + multi_bit + - classical_documentation + - multi_bit_documentation + - classical_documentation + multi_bit_documentation + +permissions: {} + +jobs: + run-benchmarks: + name: benchmark_cpu/run-benchmarks + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: ${{ inputs.command }} + op_flavor: ${{ inputs.op_flavor }} + bench_type: ${{ inputs.bench_type }} + params_type: ${{ inputs.params_type }} + all_precisions: ${{ inputs.all_precisions }} + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} diff --git a/.github/workflows/benchmark_integer.yml b/.github/workflows/benchmark_cpu_common.yml similarity index 52% rename from .github/workflows/benchmark_integer.yml rename to .github/workflows/benchmark_cpu_common.yml index d8473a39f..71cba434e 100644 --- a/.github/workflows/benchmark_integer.yml +++ b/.github/workflows/benchmark_cpu_common.yml @@ -1,28 +1,48 @@ -# Run all integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_integer +# Run benchmarks on an instance and return parsed results to Slab CI bot. +name: benchmark_cpu_common on: - workflow_dispatch: + workflow_call: inputs: + command: # Any make recipes stripped of the "bench_" prefix in the Makefile + type: string # Use comma separated values to generate an array + required: true + op_flavor: + type: string # Use comma separated values to generate an array + default: default + bench_type: + type: string + default: latency + params_type: + type: string + default: classical all_precisions: - description: "Run all precisions" type: boolean default: false - bench_type: - description: "Benchmarks type" - type: choice - default: latency - options: - - latency - - throughput - - both - - schedule: - # Weekly benchmarks will be triggered each Saturday at 1a.m. - - cron: '0 1 * * 6' - # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. - # These benchmarks are far longer to execute hence the reason to run them only four time a year. - - cron: '0 4 25 MAR,JUN,SEP,DEC *' + additional_recipe: # Make recipes to run aside the benchmarks. + type: string # Use comma separated values to generate an array + additional_file_to_parse: # Make recipes aditional to run aside the benchamrks. Use comma separated values to generate an array + type: string # Use comma separated values to generate an array + additional_results_type: + type: string + default: object-size + secrets: + REPO_CHECKOUT_TOKEN: + required: true + SLAB_ACTION_TOKEN: + required: true + SLAB_BASE_URL: + required: true + SLAB_URL: + required: true + JOB_SECRET: + required: true + SLACK_CHANNEL: + required: true + BOT_USERNAME: + required: true + SLACK_WEBHOOK: + required: true env: CARGO_TERM_COLOR: always @@ -36,58 +56,58 @@ env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} FAST_BENCH: TRUE - permissions: {} jobs: prepare-matrix: - name: benchmark_integer/prepare-matrix + name: benchmark_cpu_common/prepare-matrix runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') outputs: - op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} + command: ${{ steps.set_matrix_args.outputs.command }} + op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} + bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} + params_type: ${{ steps.set_matrix_args.outputs.params_type }} + env: + INPUTS_COMMAND: ${{ inputs.command }} + INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} steps: - - name: Weekly benchmarks - if: github.event.schedule == '0 1 * * 6' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" + - name: Parse user inputs + shell: python + run: | # zizmor: ignore[template-injection] these env variables are safe + split_command = "${{ inputs.command }}".replace(" ", "").split(",") + split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",") - - name: Quarterly benchmarks - if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' - run: | - echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\", \"misc\"]" >> "${GITHUB_ENV}" + if "${{ inputs.bench_type }}" == "both": + bench_type = ["latency", "throughput"] + else: + bench_type = ["${{ inputs.bench_type }}", ] - - name: Set benchmark types - if: github.event_name == 'workflow_dispatch' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} + if "+" in "${{ inputs.params_type }}": + split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+") + else: + split_params_type = ["${{ inputs.params_type }}", ] - - name: Default benchmark type - if: github.event_name != 'workflow_dispatch' - run: | - echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" + with open("${{ github.env }}", "a") as f: + for env_name, values_to_join in [ + ("COMMAND", split_command), + ("OP_FLAVOR", split_op_flavor), + ("BENCH_TYPE", bench_type), + ("PARAMS_TYPE", split_params_type), + ]: + f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""") - - name: Set operation flavor output - id: set_op_flavor - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" + - name: Set martix arguments outputs + id: set_matrix_args + run: | # zizmor: ignore[template-injection] these env variable are safe + { + echo "command=${{ toJSON(env.COMMAND) }}"; + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}"; + echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}"; + echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}"; + } >> "${GITHUB_OUTPUT}" setup-instance: - name: benchmark_integer/setup-instance + name: benchmark_cpu_common/setup-instance needs: prepare-matrix runs-on: ubuntu-latest outputs: @@ -105,19 +125,17 @@ jobs: profile: bench integer-benchmarks: - name: benchmark_integer/integer-benchmarks + name: benchmark_cpu_common/integer-benchmarks needs: [ prepare-matrix, setup-instance ] runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} timeout-minutes: 1440 # 24 hours strategy: max-parallel: 1 matrix: - command: [ integer, integer_multi_bit] - op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} + command: ${{ fromJSON(needs.prepare-matrix.outputs.command) }} + op_flavor: ${{ fromJSON(needs.prepare-matrix.outputs.op_flavor) }} bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} + params_type: ${{ fromJSON(needs.prepare-matrix.outputs.params_type) }} steps: - name: Checkout tfhe-rs repo with tags uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 @@ -142,14 +160,6 @@ jobs: with: toolchain: nightly - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - name: Should run benchmarks with all precisions if: inputs.all_precisions run: | @@ -157,20 +167,13 @@ jobs: - name: Run benchmarks with AVX512 run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" bench_"${BENCH_COMMAND}" + make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}" env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} + BENCH_PARAMS_TYPE: ${{ matrix.params_type }} BENCH_COMMAND: ${{ matrix.command }} - # Run these benchmarks only once per benchmark type - - name: Run compression benchmarks with AVX512 - if: matrix.op_flavor == 'default' && matrix.command == 'integer' - run: | - make BENCH_TYPE="${BENCH_TYPE}" bench_integer_compression - env: - BENCH_TYPE: ${{ matrix.bench_type }} - - name: Parse results run: | python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ @@ -187,12 +190,45 @@ jobs: REF_NAME: ${{ github.ref_name }} BENCH_TYPE: ${{ matrix.bench_type }} + - name: Run additional benchmarks + if: ${{ inputs.additional_recipe }} + run: | + targets_list="${targets}" + IFS=',' + for target in $targets_list; do + make "$target" + done + env: + targets: ${{ inputs.additional_recipe }} + + - name: Parse additional benchmarks results files + if: ${{ inputs.additional_file_to_parse }} + run: | + filenames_list="${filenames}" + IFS=',' + for filename in $filenames_list; do + python3 ./ci/benchmark_parser.py "tfhe-benchmark/${filename}" "${RESULTS_FILENAME}" \ + --"${results_type}" \ + --append-results + done + env: + filenames: ${{ inputs.additional_file_to_parse }} + results_type: ${{ inputs.additional_results_type }} + - name: Upload parsed results artifact uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 with: - name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }}_${{ matrix.bench_type }} + name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }}_${{ matrix.bench_type }}_${{ matrix.params_type }} path: ${{ env.RESULTS_FILENAME }} + - name: Checkout Slab repo + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + repository: zama-ai/slab + path: slab + persist-credentials: 'false' + token: ${{ secrets.REPO_CHECKOUT_TOKEN }} + - name: Send data to Slab shell: bash run: | @@ -203,15 +239,15 @@ jobs: SLAB_URL: ${{ secrets.SLAB_URL }} - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} + if: ${{ failure() }} continue-on-error: true uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 env: SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "CPU bencmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" teardown-instance: - name: benchmark_integer/teardown-instance + name: benchmark_cpu_common/teardown-instance if: ${{ always() && needs.setup-instance.result == 'success' }} needs: [ setup-instance, integer-benchmarks ] runs-on: ubuntu-latest @@ -232,4 +268,4 @@ jobs: uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 env: SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + SLACK_MESSAGE: "Instance teardown (cpu-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_cpu_weekly.yml b/.github/workflows/benchmark_cpu_weekly.yml new file mode 100644 index 000000000..ce8aa770f --- /dev/null +++ b/.github/workflows/benchmark_cpu_weekly.yml @@ -0,0 +1,220 @@ +# Run CPU latencies benchmarks AWS VMs and return parsed results to Slab CI bot. +name: benchmark_cpu_weekly + +on: + schedule: + # Weekly schedules are separated in two groups to avoid spawning too many the machines at once thus risking resource shortages. + # Group 1 + # ------- + # Weekly benchmarks will be triggered each Saturday at 1a.m. + - cron: '0 1 * * 6' + # Group 2 + # ------- + # Weekly benchmarks will be triggered each Sunday at 3a.m. + - cron: '0 3 * * 0' + + # Quarterly benchmarks will be triggered right before the end of the quarter, the 25th of the current month at 4a.m. + # These benchmarks are far longer to execute, hence the reason to run them only four times a year. + - cron: '0 4 25 MAR,JUN,SEP,DEC *' + +permissions: {} + +jobs: + prepare-inputs: + name: benchmark_cpu_weekly/prepare-inputs + runs-on: ubuntu-latest + outputs: + is_weekly_bench_group_1: ${{ steps.check_bench_group_1.outputs.is_weekly_bench_group_1 }} + is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }} + is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }} + op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} + all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }} + steps: + - name: Check is weekly bench group 1 + id: check_bench_group_1 + run: | # zizmor: ignore[template-injection] this env variable is safe + echo "is_weekly_bench_group_1=${{ github.event.schedule == '0 1 * * 6' }}" >> "${GITHUB_OUTPUT}" + + - name: Check is weekly bench group 2 + id: check_bench_group_2 + run: | # zizmor: ignore[template-injection] this env variable is safe + echo "is_weekly_bench_group_2=${{ github.event.schedule == '0 3 * * 0' }}" >> "${GITHUB_OUTPUT}" + + - name: Check is quarterly bench + id: check_quarterly_bench + run: | # zizmor: ignore[template-injection] this env variable is safe + echo "is_quarterly_bench=${{ github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' }}" >> "${GITHUB_OUTPUT}" + + - name: Weekly benchmarks + if: env.WEEKLY_BENCH_GROUP_1 || env.WEEKLY_BENCH_GROUP_2 + run: | + echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" + echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}" + + - name: Quarterly benchmarks + if: env.QUARTERLY_BENCH + run: | + echo "OP_FLAVOR=[\"default\", \"unchecked\"]" >> "${GITHUB_ENV}" + echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}" + + - name: Set operation flavor output + id: set_op_flavor + run: | # zizmor: ignore[template-injection] this env variable is safe + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" + + - name: Set bit precisions output + id: set_all_precisions + run: | # zizmor: ignore[template-injection] this env variable is safe + echo "all_precisions=${{ toJSON(env.ALL_PRECISIONS) }}" >> "${GITHUB_OUTPUT}" + + run-benchmarks-integer: + name: benchmark_gpu_weekly/run-benchmarks-integer + if: github.repository == 'zama-ai/tfhe-rs' + && (needs.prepare-inputs.outputs.is_weekly_bench_group_1 || needs.prepare-inputs.outputs.is_quarterly_bench) + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: integer,signed_integer, integer_compression + op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }} + all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }} + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-integer-zk-pke: + name: benchmark_gpu_weekly/run-benchmarks-integer-zk-pke + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_1 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: integer_zk + additional_file_to_parse: pke_zk_crs_sizes.csv + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-hlapi-erc20: + name: benchmark_gpu_weekly/run-benchmarks-hlapi-erc20 + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_2 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: hlapi_erc20 + additional_file_to_parse: erc20_pbs_count.csv + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-hlapi-dex: + name: benchmark_gpu_weekly/run-benchmarks-hlapi-dex + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_1 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: hlapi_dex + additional_file_to_parse: dex_swap_request_update_dex_balance_pbs_count.csv,dex_swap_request_finalize_pbs_count.csv,dex_swap_claim_prepare_pbs_count.csv,dex_swap_claim_update_dex_balance_pbs_count.csv + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-core-crypto: + name: benchmark_gpu_weekly/run-benchmarks-core-crypto + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_1 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: ks,pbs,pbs128,ks_pbs + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-shortint: + name: benchmark_gpu_weekly/run-benchmarks-shortint + if: github.repository == 'zama-ai/tfhe-rs' + && (needs.prepare-inputs.outputs.is_weekly_bench_group_2 || needs.prepare-inputs.outputs.is_quarterly_bench) + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }} + command: shortint + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-boolean: + name: benchmark_gpu_weekly/run-benchmarks-boolean + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_2 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: boolean + additional_recipe: measure_boolean_key_sizes + additional_file_to_parse: boolean_key_size.csv + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-tfhe-zk-pok: + name: benchmark_gpu_weekly/run-benchmarks-tfhe-zk-pok + if: github.repository == 'zama-ai/tfhe-rs' + && needs.prepare-inputs.outputs.is_weekly_bench_group_1 + needs: prepare-inputs + uses: ./.github/workflows/benchmark_cpu_common.yml + with: + command: tfhe_zk_pok + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} diff --git a/.github/workflows/benchmark_dex.yml b/.github/workflows/benchmark_dex.yml deleted file mode 100644 index b6ef626da..000000000 --- a/.github/workflows/benchmark_dex.yml +++ /dev/null @@ -1,170 +0,0 @@ -# Run all DEX benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_dex - -on: - workflow_dispatch: - schedule: - # Weekly benchmarks will be triggered each Saturday at 5a.m. - - cron: '0 5 * * 6' - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - -permissions: {} - -jobs: - setup-instance: - name: benchmark_dex/setup-instance - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - dex-benchmarks: - name: benchmark_dex/dex-benchmarks - needs: setup-instance - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - timeout-minutes: 720 # 12 hours - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Run benchmarks - run: | - make bench_hlapi_dex - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 - env: - REF_NAME: ${{ github.ref_name }} - - - name: Parse swap request update PBS counts - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/dex_swap_request_update_dex_balance_pbs_count.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Parse swap request finalize PBS counts - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/dex_swap_request_finalize_pbs_count.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Parse swap claim prepare PBS counts - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/dex_swap_claim_prepare_pbs_count.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Parse swap claim update PBS counts - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/dex_swap_claim_update_dex_balance_pbs_count.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_dex - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "DEX benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_dex/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, dex-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (dex-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_erc20.yml b/.github/workflows/benchmark_erc20.yml deleted file mode 100644 index 6d7b146ff..000000000 --- a/.github/workflows/benchmark_erc20.yml +++ /dev/null @@ -1,153 +0,0 @@ -# Run all ERC20 benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_erc20 - -on: - workflow_dispatch: - schedule: - # Weekly benchmarks will be triggered each Saturday at 5a.m. - - cron: '0 5 * * 6' - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - -permissions: {} - -jobs: - setup-instance: - name: benchmark_erc20/setup-instance - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - erc20-benchmarks: - name: benchmark_erc20/erc20-benchmarks - needs: setup-instance - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - timeout-minutes: 720 # 12 hours - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Run benchmarks - run: | - make bench_hlapi_erc20 - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 - env: - REF_NAME: ${{ github.ref_name }} - - - name: Parse PBS counts - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/erc20_pbs_count.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_erc20 - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "ERC20 benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_erc20/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, erc20-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (erc20-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_shortint.yml b/.github/workflows/benchmark_shortint.yml deleted file mode 100644 index a883ec4ca..000000000 --- a/.github/workflows/benchmark_shortint.yml +++ /dev/null @@ -1,179 +0,0 @@ -# Run all shortint benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_shortint - -on: - workflow_dispatch: - schedule: - # Weekly benchmarks will be triggered each Saturday at 1a.m. - - cron: '0 1 * * 6' - # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. - # These benchmarks are far longer to execute hence the reason to run them only four time a year. - - cron: '0 4 25 MAR,JUN,SEP,DEC *' - - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - -permissions: {} - -jobs: - prepare-matrix: - name: benchmark_shortint/prepare-matrix - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - steps: - - name: Weekly benchmarks - if: github.event_name == 'workflow_dispatch' || - github.event.schedule == '0 1 * * 6' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - - - name: Quarterly benchmarks - if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' - run: | - echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\"]" >> "${GITHUB_ENV}" - - - name: Set operation flavor output - id: set_op_flavor - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" - - setup-instance: - name: benchmark_shortint/setup-instance - needs: prepare-matrix - runs-on: ubuntu-latest - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - shortint-benchmarks: - name: benchmark_shortint/shortint-benchmarks - needs: [ prepare-matrix, setup-instance ] - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - strategy: - max-parallel: 1 - matrix: - op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Run benchmarks with AVX512 - run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" bench_shortint - env: - OP_FLAVOR: ${{ matrix.op_flavor }} - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 - env: - REF_NAME: ${{ github.ref_name }} - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_shortint_${{ matrix.op_flavor }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Shortint full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_shortint/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, shortint-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (shortint-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_signed_integer.yml b/.github/workflows/benchmark_signed_integer.yml deleted file mode 100644 index 0cca964c2..000000000 --- a/.github/workflows/benchmark_signed_integer.yml +++ /dev/null @@ -1,227 +0,0 @@ -# Run all signed integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: benchmark_signed_integer - -on: - workflow_dispatch: - inputs: - all_precisions: - description: "Run all precisions" - type: boolean - default: false - bench_type: - description: "Benchmarks type" - type: choice - default: latency - options: - - latency - - throughput - - both - - schedule: - # Weekly benchmarks will be triggered each Saturday at 1a.m. - - cron: '0 1 * * 6' - # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. - # These benchmarks are far longer to execute hence the reason to run them only four time a year. - - cron: '0 4 25 MAR,JUN,SEP,DEC *' - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE - - -permissions: {} - -jobs: - prepare-matrix: - name: benchmark_signed_integer/prepare-matrix - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} - steps: - - name: Weekly benchmarks - if: github.event.schedule == '0 1 * * 6' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - - - name: Quarterly benchmarks - if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' - run: | - echo "OP_FLAVOR=[\"default\", \"unchecked\"]" >> "${GITHUB_ENV}" - - - name: Set benchmark types - if: github.event_name == 'workflow_dispatch' - run: | - echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} - - - name: Default benchmark type - if: github.event_name != 'workflow_dispatch' - run: | - echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" - - - name: Set operation flavor output - id: set_op_flavor - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" - - setup-instance: - name: benchmark_signed_integer/setup-instance - needs: prepare-matrix - runs-on: ubuntu-latest - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - signed-integer-benchmarks: - name: benchmark_signed_integer/signed-integer-benchmarks - needs: [ prepare-matrix, setup-instance ] - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - concurrency: - group: ${{ github.workflow_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - timeout-minutes: 1440 # 24 hours - strategy: - max-parallel: 1 - matrix: - command: [ integer, integer_multi_bit ] - op_flavor: ${{ fromJSON(needs.prepare-matrix.outputs.op_flavor) }} - bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - - name: Run benchmarks with AVX512 - run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" bench_signed_"${BENCH_COMMAND}" - env: - OP_FLAVOR: ${{ matrix.op_flavor }} - BENCH_TYPE: ${{ matrix.bench_type }} - BENCH_COMMAND: ${{ matrix.command }} - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --bench-type "${BENCH_TYPE}" - env: - REF_NAME: ${{ github.ref_name }} - BENCH_TYPE: ${{ matrix.bench_type }} - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }}_${{ matrix.bench_type }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Signed integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_signed_integer/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, signed-integer-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (signed-integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_tfhe_zk_pok.yml b/.github/workflows/benchmark_tfhe_zk_pok.yml deleted file mode 100644 index e1793f2a9..000000000 --- a/.github/workflows/benchmark_tfhe_zk_pok.yml +++ /dev/null @@ -1,198 +0,0 @@ -# Run benchmarks of the tfhe-zk-pok crate on an instance and return parsed results to Slab CI bot. -name: benchmark_tfhe_zk_pok - -on: - workflow_dispatch: - inputs: - bench_type: - description: "Benchmarks type" - type: choice - default: latency - options: - - latency - - throughput - push: - branches: - - main - schedule: - # Weekly benchmarks will be triggered each Saturday at 3a.m. - - cron: '0 3 * * 6' -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - BENCH_TYPE: ${{ inputs.bench_type || 'latency' }} - - -permissions: {} - -jobs: - should-run: - name: benchmark_tfhe_zk_pok/should-run - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || - ((github.event_name == 'push' || github.event_name == 'schedule') && github.repository == 'zama-ai/tfhe-rs') - outputs: - zk_pok_changed: ${{ steps.changed-files.outputs.zk_pok_any_changed }} - steps: - - name: Checkout tfhe-rs - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Check for file changes - id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files_yaml: | - zk_pok: - - tfhe-zk-pok/** - - .github/workflows/benchmark_tfhe_zk_pok.yml - - setup-instance: - name: benchmark_tfhe_zk_pok/setup-instance - runs-on: ubuntu-latest - needs: should-run - if: github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || - (github.event_name == 'push' && - github.repository == 'zama-ai/tfhe-rs' && - needs.should-run.outputs.zk_pok_changed == 'true') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - tfhe-zk-pok-benchmarks: - name: benchmark_tfhe_zk_pok/tfhe-zk-pok-benchmarks - if: needs.setup-instance.result != 'skipped' - needs: setup-instance - concurrency: - group: ${{ github.workflow_ref }}_${{github.event_name}}${{ github.ref == 'refs/heads/main' && github.sha || '' }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Run benchmarks - run: | - make BENCH_TYPE="${BENCH_TYPE}" bench_tfhe_zk_pok - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --crate tfhe-zk-pok \ - --hardware "hpc7a.96xlarge" \ - --backend cpu \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --bench-type "${BENCH_TYPE}" - env: - REF_NAME: ${{ github.ref_name }} - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_tfhe_zk_pok_${{ env.BENCH_TYPE }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "tfhe-zk-pok benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_tfhe_zk_pok/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, tfhe-zk-pok-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (tfhe-zk-pok-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/benchmark_zk_pke.yml b/.github/workflows/benchmark_zk_pke.yml deleted file mode 100644 index b68eafa85..000000000 --- a/.github/workflows/benchmark_zk_pke.yml +++ /dev/null @@ -1,248 +0,0 @@ -# Run PKE Zero-Knowledge benchmarks on an instance and return parsed results to Slab CI bot. -name: benchmark_zk_pke - -on: - workflow_dispatch: - inputs: - bench_type: - description: "Benchmarks type" - type: choice - default: latency - options: - - latency - - throughput - - both - - push: - branches: - - main - schedule: - # Weekly benchmarks will be triggered each Saturday at 3a.m. - - cron: '0 3 * * 6' -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - -permissions: {} - -jobs: - should-run: - name: benchmark_zk_pke/should-run - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || - ((github.event_name == 'push' || github.event_name == 'schedule') && github.repository == 'zama-ai/tfhe-rs') - outputs: - zk_pok_changed: ${{ steps.changed-files.outputs.zk_pok_any_changed }} - steps: - - name: Checkout tfhe-rs - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Check for file changes - id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files_yaml: | - zk_pok: - - tfhe/Cargo.toml - - tfhe-csprng/** - - tfhe-fft/** - - tfhe-zk-pok/** - - tfhe/src/core_crypto/** - - tfhe/src/shortint/** - - tfhe/src/integer/** - - tfhe/src/zk.rs - - tfhe/benches/integer/zk_pke.rs - - .github/workflows/zk_pke_benchmark.yml - - prepare-matrix: - name: benchmark_zk_pke/prepare-matrix - runs-on: ubuntu-latest - if: github.event_name != 'schedule' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') - outputs: - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} - steps: - - name: Set benchmark types - if: github.event_name == 'workflow_dispatch' - run: | - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} - - - name: Default benchmark type - if: github.event_name != 'workflow_dispatch' - run: | - echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" - - setup-instance: - name: benchmark_zk_pke/setup-instance - runs-on: ubuntu-latest - needs: [ should-run, prepare-matrix ] - if: github.event_name == 'workflow_dispatch' || - (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || - (github.event_name == 'push' && - github.repository == 'zama-ai/tfhe-rs' && - needs.should-run.outputs.zk_pok_changed == 'true') - outputs: - runner-name: ${{ steps.start-instance.outputs.label }} - steps: - - name: Start instance - id: start-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: start - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - backend: aws - profile: bench - - pke-zk-benchmarks: - name: benchmark_zk_pke/pke-zk-benchmarks - if: needs.setup-instance.result != 'skipped' - needs: [ prepare-matrix, setup-instance ] - concurrency: - group: ${{ github.workflow_ref }}_${{github.event_name}}${{ github.ref == 'refs/heads/main' && github.sha || '' }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - strategy: - max-parallel: 1 - matrix: - bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} - steps: - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Run benchmarks with AVX512 - run: | - make BENCH_TYPE="${BENCH_TYPE}" bench_integer_zk - env: - BENCH_TYPE: ${{ matrix.bench_type }} - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpc7a.96xlarge" \ - --backend cpu \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --bench-type "${BENCH_TYPE}" - env: - REF_NAME: ${{ github.ref_name }} - BENCH_TYPE: ${{ matrix.bench_type }} - - - name: Parse CRS sizes results - run: | - python3 ./ci/benchmark_parser.py tfhe-benchmark/pke_zk_crs_sizes.csv "${RESULTS_FILENAME}" \ - --object-sizes \ - --append-results - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_integer_zk_${{ matrix.bench_type }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() || (cancelled() && github.event_name != 'pull_request') }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "PKE ZK benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - teardown-instance: - name: benchmark_zk_pke/teardown-instance - if: ${{ always() && needs.setup-instance.result == 'success' }} - needs: [ setup-instance, pke-zk-benchmarks ] - runs-on: ubuntu-latest - steps: - - name: Stop instance - id: stop-instance - uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac - with: - mode: stop - github-token: ${{ secrets.SLAB_ACTION_TOKEN }} - slab-url: ${{ secrets.SLAB_BASE_URL }} - job-secret: ${{ secrets.JOB_SECRET }} - label: ${{ needs.setup-instance.outputs.runner-name }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "Instance teardown (pke-zk-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/Makefile b/Makefile index 5536a6210..73762569b 100644 --- a/Makefile +++ b/Makefile @@ -1341,14 +1341,14 @@ print_doc_bench_parameters: .PHONY: bench_integer # Run benchmarks for unsigned integer bench_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- .PHONY: bench_signed_integer # Run benchmarks for signed integer bench_signed_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --