From b3c36475301294bbaf2d8f8871191e8d64ce777c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Thu, 20 Nov 2025 18:24:27 +0100 Subject: [PATCH] chore(ci): add workflow to update documentation benchmark tables This new workflow can trigger all the required benchmarks needed to populate benchmarks tables in documentation. It also can generate SVG tables and store them as artifacts. Optionally, it can open a pull-request to update the current tables in documentation. --- .github/workflows/benchmark_cpu.yml | 14 +- .github/workflows/benchmark_cpu_common.yml | 18 +- .github/workflows/benchmark_cpu_weekly.yml | 12 +- .github/workflows/benchmark_documentation.yml | 207 +++++++++++++++++- .github/workflows/benchmark_gpu.yml | 14 +- .github/workflows/benchmark_gpu_4090.yml | 3 +- .github/workflows/benchmark_gpu_common.yml | 118 ++++------ .github/workflows/benchmark_gpu_weekly.yml | 8 +- .github/workflows/benchmark_hpu.yml | 14 +- .github/workflows/benchmark_hpu_common.yml | 15 +- .github/workflows/generate_svg_common.yml | 80 ++++++- .github/workflows/generate_svgs.yml | 175 ++++++++++++++- Makefile | 22 +- tfhe-benchmark/src/utilities.rs | 42 +++- 14 files changed, 572 insertions(+), 170 deletions(-) diff --git a/.github/workflows/benchmark_cpu.yml b/.github/workflows/benchmark_cpu.yml index 3e2d5d37a..01838c043 100644 --- a/.github/workflows/benchmark_cpu.yml +++ b/.github/workflows/benchmark_cpu.yml @@ -34,10 +34,14 @@ on: - smart - unchecked - misc - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -71,7 +75,7 @@ jobs: op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} params_type: ${{ inputs.params_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_cpu_common.yml b/.github/workflows/benchmark_cpu_common.yml index 39ddb6ecb..12c9a97d3 100644 --- a/.github/workflows/benchmark_cpu_common.yml +++ b/.github/workflows/benchmark_cpu_common.yml @@ -16,9 +16,9 @@ on: params_type: type: string default: classical - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast additional_recipe: # Make recipes to run aside the benchmarks. type: string # Use comma separated values to generate an array additional_file_to_parse: # Other files to parse, located under tfhe-benchmark/ directory @@ -54,7 +54,6 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE permissions: {} @@ -69,9 +68,6 @@ jobs: op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} params_type: ${{ steps.set_matrix_args.outputs.params_type }} - env: - INPUTS_COMMAND: ${{ inputs.command }} - INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} steps: - name: Parse user inputs shell: python @@ -162,19 +158,15 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Run benchmarks with AVX512 run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}" + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}" env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_PARAMS_TYPE: ${{ matrix.params_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/benchmark_cpu_weekly.yml b/.github/workflows/benchmark_cpu_weekly.yml index ad44465ae..0a9138e87 100644 --- a/.github/workflows/benchmark_cpu_weekly.yml +++ b/.github/workflows/benchmark_cpu_weekly.yml @@ -30,7 +30,7 @@ jobs: is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }} is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }} op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }} + precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }} steps: - name: Check is weekly bench group 1 id: check_bench_group_1 @@ -52,13 +52,13 @@ jobs: steps.check_bench_group_2.outputs.is_weekly_bench_group_2 == 'true' run: | echo "OP_FLAVOR=default" >> "${GITHUB_ENV}" - echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}" + echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}" - name: Quarterly benchmarks if: steps.check_quarterly_bench.outputs.is_quarterly_bench == 'true' run: | echo "OP_FLAVOR=\"default,unchecked\"" >> "${GITHUB_ENV}" - echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}" + echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}" - name: Set operation flavor output id: set_op_flavor @@ -66,9 +66,9 @@ jobs: echo "op_flavor=${{ env.OP_FLAVOR }}" >> "${GITHUB_OUTPUT}" - name: Set bit precisions output - id: set_all_precisions + id: set_precisions_set run: | # zizmor: ignore[template-injection] this env variable is safe - echo "all_precisions=${{ env.ALL_PRECISIONS }}" >> "${GITHUB_OUTPUT}" + echo "precisions_set=${{ env.PRECISIONS_SET }}" >> "${GITHUB_OUTPUT}" run-benchmarks-integer: name: benchmark_cpu_weekly/run-benchmarks-integer @@ -79,7 +79,7 @@ jobs: with: command: integer,signed_integer, integer_compression op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }} - all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }} + precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_documentation.yml b/.github/workflows/benchmark_documentation.yml index c4da7112f..43c028540 100644 --- a/.github/workflows/benchmark_documentation.yml +++ b/.github/workflows/benchmark_documentation.yml @@ -1,17 +1,212 @@ -name: benchmark_documentation.yml +# Run all benchmarks displayed in the public documentation. +name: benchmark_documentation on: workflow_dispatch: + inputs: + run-cpu-benchmarks: + description: "Run CPU benchmarks" + type: boolean + default: true + run-gpu-benchmarks: + description: "Run GPU benchmarks" + type: boolean + default: true + run-hpu-benchmarks: + description: "Run HPU benchmarks" + type: boolean + default: true + generate-svgs: + description: "Generate SVG tables" + type: boolean + default: true + open-pr: + description: "Open a PR with the benchmark results" + type: boolean + default: false permissions: {} # zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow jobs: - placeholder: - name: benchmark_documentation/placeholder - runs-on: ubuntu-latest + run-benchmarks-cpu-integer: + name: benchmark_documentation/run-benchmarks-cpu-integer + uses: ./.github/workflows/benchmark_cpu_common.yml + if: inputs.run-cpu-benchmarks + with: + command: integer + op_flavor: fast_default + bench_type: both + precisions_set: documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + run-benchmarks-gpu-integer: + name: benchmark_documentation/run-benchmarks-gpu-integer + uses: ./.github/workflows/benchmark_gpu_common.yml + if: inputs.run-gpu-benchmarks + with: + profile: multi-h100-sxm5 + hardware_name: n3-H100-SXM5x8 + command: integer_multi_bit + op_flavor: fast_default + bench_type: both + precisions_set: documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-hpu-integer: + name: benchmark_documentation/run-benchmarks-hpu-integer + uses: ./.github/workflows/benchmark_hpu_common.yml + if: inputs.run-hpu-benchmarks + with: + command: integer + op_flavor: default + bench_type: both + precisions_set: documentation + v80_pcie_dev: 24 + v80_serial_number: XFL12NWY3ZKG + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + + run-benchmarks-cpu-core-crypto: + name: benchmark_documentation/run-benchmarks-cpu-core-crypto + uses: ./.github/workflows/benchmark_cpu_common.yml + if: inputs.run-cpu-benchmarks + with: + command: pbs, ks_pbs + bench_type: latency + params_type: classical_documentation + multi_bit_documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-gpu-core-crypto: + name: benchmark_documentation/run-benchmarks-gpu-core-crypto + uses: ./.github/workflows/benchmark_gpu_common.yml + if: inputs.run-gpu-benchmarks + with: + profile: multi-h100-sxm5 + hardware_name: n3-H100-SXM5x8 + command: pbs, ks_pbs + bench_type: latency + params_type: classical_documentation + multi_bit_documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + generate-svgs-with-benchmarks-run: + name: benchmark-documentation/generate-svgs-with-benchmarks-run + if: ${{ always() && + (inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) && + inputs.generate-svgs }} + needs: [ + run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer, + run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto + ] + uses: ./.github/workflows/generate_svgs.yml + with: + time_span_days: 5 + generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }} + generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }} + generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }} + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + generate-svgs-without-benchmarks-run: + name: benchmark-documentation/generate-svgs-without-benchmarks-run + if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) && + inputs.generate-svgs }} + uses: ./.github/workflows/generate_svgs.yml + with: + time_span_days: 60 + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + open-pr: + name: benchmark-documentation/open-pr + needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ] + if: ${{ always() && inputs.open-pr && + (needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }} + runs-on: ubuntu-latest + permissions: + contents: write # Needed to create a commit + pull-requests: write # Needed to open a pull-request + env: + PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets steps: - - run: | - echo "Hello this is a placeholder workflow" + - name: Checkout tfhe-rs + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + + - name: Download SVG tables + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + path: svg_tables + merge-multiple: 'true' + + - name: Copy SVG tables to documentation location + run: | + cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}" + cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}" + + - name: Get current date + id: get-date + run: | + echo "date=$(date '+%g_%m_%d_%Hh%Mm%Ss')" >> "${GITHUB_OUTPUT}" + + - name: Create pull-request + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + sign-commits: true # Commit will be signed by github-actions bot + add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg + branch: gh-bot/docs/update-svg-tables-${{ steps.get-date.outputs.date }} + commit-message: | + chore(docs): update benchmark results for all backends + + Automated documentation update from tfhe-rs CI pipeline. + title: | + [CI] chore(docs): update benchmark results for all backends + body: | + Documentation update triggered by GitHub workflow. + labels: documentation diff --git a/.github/workflows/benchmark_gpu.yml b/.github/workflows/benchmark_gpu.yml index 4fe370b15..e20fa9e37 100644 --- a/.github/workflows/benchmark_gpu.yml +++ b/.github/workflows/benchmark_gpu.yml @@ -42,10 +42,14 @@ on: - default - fast_default - unchecked - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -108,7 +112,7 @@ jobs: op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} params_type: ${{ inputs.params_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_gpu_4090.yml b/.github/workflows/benchmark_gpu_4090.yml index 1b1479f5c..55f2ea7f3 100644 --- a/.github/workflows/benchmark_gpu_4090.yml +++ b/.github/workflows/benchmark_gpu_4090.yml @@ -11,7 +11,7 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE + BIT_SIZES_SET: FAST on: # Allows you to run this workflow manually from the Actions tab as an alternative. @@ -53,7 +53,6 @@ jobs: echo "BENCH_DATE=$(date --iso-8601=seconds)"; echo "COMMIT_DATE=${COMMIT_DATE}"; echo "COMMIT_HASH=$(git describe --tags --dirty)"; - echo "FAST_BENCH=TRUE"; } >> "${GITHUB_ENV}" env: SHA: ${{ github.sha }} diff --git a/.github/workflows/benchmark_gpu_common.yml b/.github/workflows/benchmark_gpu_common.yml index 6716b87fa..e10bd4346 100644 --- a/.github/workflows/benchmark_gpu_common.yml +++ b/.github/workflows/benchmark_gpu_common.yml @@ -25,9 +25,9 @@ on: params_type: type: string default: multi_bit - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast secrets: REPO_CHECKOUT_TOKEN: required: true @@ -56,8 +56,6 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE - permissions: {} @@ -68,79 +66,45 @@ jobs: name: benchmark_gpu_common/prepare-matrix runs-on: ubuntu-latest outputs: - command: ${{ steps.set_command.outputs.command }} - op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} - params_type: ${{ steps.set_params_type.outputs.params_type }} - env: - INPUTS_COMMAND: ${{ inputs.command }} - INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} + command: ${{ steps.set_matrix_args.outputs.command }} + op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} + bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} + params_type: ${{ steps.set_matrix_args.outputs.params_type }} steps: - - name: Set single command - if: ${{ !contains(inputs.command, ',')}} - run: | - echo "COMMAND=[\"${INPUTS_COMMAND}\"]" >> "${GITHUB_ENV}" + - name: Parse user inputs + shell: python + run: | # zizmor: ignore[template-injection] these env variables are safe + split_command = "${{ inputs.command }}".replace(" ", "").split(",") + split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",") - - name: Set multiple commands - if: ${{ contains(inputs.command, ',')}} - run: | - # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern. - # shellcheck disable=SC2001 - PARSED_COMMAND=$(echo "${INPUTS_COMMAND}" | sed 's/[[:space:]]*,[[:space:]]*/\", \"/g') - echo "COMMAND=[\"${PARSED_COMMAND}\"]" >> "${GITHUB_ENV}" + if "${{ inputs.bench_type }}" == "both": + bench_type = ["latency", "throughput"] + else: + bench_type = ["${{ inputs.bench_type }}", ] - - name: Set single operations flavor - if: ${{ !contains(inputs.op_flavor, ',')}} - run: | - echo "OP_FLAVOR=[\"${INPUTS_OP_FLAVOR}\"]" >> "${GITHUB_ENV}" + if "+" in "${{ inputs.params_type }}": + split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+") + else: + split_params_type = ["${{ inputs.params_type }}", ] - - name: Set multiple operations flavors - if: ${{ contains(inputs.op_flavor, ',')}} - run: | - # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern. - # shellcheck disable=SC2001 - PARSED_OP_FLAVOR=$(echo "${INPUTS_OP_FLAVOR}" | sed 's/[[:space:]]*,[[:space:]]*/", "/g') - echo "OP_FLAVOR=[\"${PARSED_OP_FLAVOR}\"]" >> "${GITHUB_ENV}" + with open("${{ github.env }}", "a") as f: + for env_name, values_to_join in [ + ("COMMAND", split_command), + ("OP_FLAVOR", split_op_flavor), + ("BENCH_TYPE", bench_type), + ("PARAMS_TYPE", split_params_type), + ]: + f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""") - - name: Set benchmark types - run: | - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} - - - name: Set parameters types - run: | - if [[ "${INPUTS_PARAMS_TYPE}" == "both" ]]; then - echo "PARAMS_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}" - else - echo "PARAMS_TYPE=[\"${INPUTS_PARAMS_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_PARAMS_TYPE: ${{ inputs.params_type }} - - - name: Set command output - id: set_command - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "command=${{ toJSON(env.COMMAND) }}" >> "${GITHUB_OUTPUT}" - - - name: Set operation flavor output - id: set_op_flavor - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" - - - name: Set parameters types output - id: set_params_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}" >> "${GITHUB_OUTPUT}" + - name: Set martix arguments outputs + id: set_matrix_args + run: | # zizmor: ignore[template-injection] these env variable are safe + { + echo "command=${{ toJSON(env.COMMAND) }}"; + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}"; + echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}"; + echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}"; + } >> "${GITHUB_OUTPUT}" setup-instance: name: benchmark_gpu_common/setup-instance @@ -275,19 +239,15 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Run benchmarks run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_PARAMS_TYPE: ${{ matrix.params_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/benchmark_gpu_weekly.yml b/.github/workflows/benchmark_gpu_weekly.yml index 9cc416c36..9b4f75755 100644 --- a/.github/workflows/benchmark_gpu_weekly.yml +++ b/.github/workflows/benchmark_gpu_weekly.yml @@ -22,7 +22,7 @@ jobs: command: integer_multi_bit op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -43,7 +43,7 @@ jobs: command: integer_compression op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -64,7 +64,7 @@ jobs: command: integer_zk,integer_aes,integer_aes256 op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -85,7 +85,7 @@ jobs: command: hlapi_noise_squash op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_hpu.yml b/.github/workflows/benchmark_hpu.yml index 630449e9d..d71c02527 100644 --- a/.github/workflows/benchmark_hpu.yml +++ b/.github/workflows/benchmark_hpu.yml @@ -19,10 +19,14 @@ on: options: - default - fast_default - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -50,7 +54,7 @@ jobs: command: ${{ inputs.command }} op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} v80_pcie_dev: ${{ inputs.v80_pcie_dev }} v80_serial_number: ${{ inputs.v80_serial_number }} secrets: diff --git a/.github/workflows/benchmark_hpu_common.yml b/.github/workflows/benchmark_hpu_common.yml index bcc089df4..1b7eea50e 100644 --- a/.github/workflows/benchmark_hpu_common.yml +++ b/.github/workflows/benchmark_hpu_common.yml @@ -13,9 +13,9 @@ on: bench_type: type: string default: latency - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast v80_pcie_dev: type: string default: 24 @@ -48,7 +48,6 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" - FAST_BENCH: TRUE permissions: {} @@ -139,11 +138,6 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Select HPU board run: | echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}" @@ -156,11 +150,12 @@ jobs: run: | echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}" make pull_hpu_files - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/generate_svg_common.yml b/.github/workflows/generate_svg_common.yml index dc6f2a60a..5900acdca 100644 --- a/.github/workflows/generate_svg_common.yml +++ b/.github/workflows/generate_svg_common.yml @@ -1,17 +1,85 @@ -name: generate_svg_common.yml +name: generate_svg_common on: workflow_call: + inputs: + backend: + type: string + required: true + hardware_name: + type: string + required: true + layer: + type: string + required: true + pbs_kind: # Valid values are 'classical', 'multi_bit' or 'any' + type: string + required: true + grouping_factor: # Valid values are 2, 3, or 4 + type: string + default: 4 + bench_type: # Valid values are 'latency', 'throughput' + type: string + required: true + time_span_days: + type: string + default: 60 + output_filename: + type: string + required: true + secrets: + DATA_EXTRACTOR_DATABASE_USER: + required: true + DATA_EXTRACTOR_DATABASE_HOST: + required: true + DATA_EXTRACTOR_DATABASE_PASSWORD: + required: true permissions: {} # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency jobs: - placeholder: - name: generate_svg_common/placeholder + generate-table: + name: generate_svg_common/generate-table runs-on: ubuntu-latest - steps: - - run: | - echo "Hello this is a placeholder workflow" + - name: Checkout tfhe-rs + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + + - name: Produce table from database + run: | + python3 -m pip install -r ci/data_extractor/requirements.txt + python3 ci/data_extractor/src/data_extractor.py "${OUTPUT_FILENAME}" \ + --generate-svg \ + --branch "${REF_NAME}" \ + --backend "${BACKEND}" \ + --hardware "${HARDWARE_NAME}" \ + --tfhe-rs-layer "${LAYER}" \ + --pbs-kind "${PBS_KIND}" \ + --grouping-factor "${GROUPING_FACTOR}" \ + --bench-type "${BENCH_TYPE}" \ + --time-span-days "${TIME_SPAN}" + env: + OUTPUT_FILENAME: ${{ inputs.output_filename }} + REF_NAME: ${{ github.ref_name }} + BACKEND: ${{ inputs.backend }} + HARDWARE_NAME: ${{ inputs.hardware_name }} + LAYER: ${{ inputs.layer }} + PBS_KIND: ${{ inputs.pbs_kind }} + GROUPING_FACTOR: ${{ inputs.grouping_factor }} + BENCH_TYPE: ${{ inputs.bench_type }} + TIME_SPAN: ${{ inputs.time_span_days }} + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + - name: Upload tables + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 + with: + name: ${{ github.sha }}_${{ inputs.backend }}_${{ inputs.layer }}_${{ inputs.pbs_kind }}_${{ inputs.bench_type }}_tables + # This will upload all the file generated + path: ${{ inputs.output_filename }}*.svg + retention-days: 60 diff --git a/.github/workflows/generate_svgs.yml b/.github/workflows/generate_svgs.yml index f84362dd5..40bb1ef1a 100644 --- a/.github/workflows/generate_svgs.yml +++ b/.github/workflows/generate_svgs.yml @@ -1,17 +1,178 @@ -name: generate_svgs.yml +# Generate benchmark SVGs for public documentation +name: generate_documentation_svgs on: workflow_call: + inputs: + time_span_days: + type: string + required: true + generate-cpu-svgs: + type: boolean + default: true + generate-gpu-svgs: + type: boolean + default: true + generate-hpu-svgs: + type: boolean + default: true + secrets: + DATA_EXTRACTOR_DATABASE_USER: + required: true + DATA_EXTRACTOR_DATABASE_HOST: + required: true + DATA_EXTRACTOR_DATABASE_PASSWORD: + required: true permissions: {} # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency jobs: - placeholder: - name: generate_svgs/placeholder - runs-on: ubuntu-latest + # ----------------------------------------------------------- + # Integer benchmarks tables + # ----------------------------------------------------------- - steps: - - run: | - echo "Hello this is a placeholder workflow" + cpu-integer-latency-table: + name: generate_documentation_svgs/cpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: integer + pbs_kind: classical + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-integer-benchmark-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + cpu-integer-throughput-table: + name: generate_documentation_svgs/cpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: integer + pbs_kind: classical + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-integer-benchmark-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-integer-latency-table: + name: generate_documentation_svgs/gpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: integer + pbs_kind: multi_bit + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-integer-throughput-table: + name: generate_documentation_svgs/gpu-integer-throughput-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: integer + pbs_kind: multi_bit + grouping_factor: 4 + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + hpu-integer-latency-table: + name: generate_documentation_svgs/hpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-hpu-svgs + with: + backend: hpu + hardware_name: hpu_x1 + layer: integer + pbs_kind: classical + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + hpu-integer-throughput-table: + name: generate_documentation_svgs/hpu-integer-throughput-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-hpu-svgs + with: + backend: hpu + hardware_name: hpu_x1 + layer: integer + pbs_kind: classical + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + # ----------------------------------------------------------- + # PBS benchmarks tables + # ----------------------------------------------------------- + + cpu-pbs-tables: + name: generate_documentation_svgs/cpu-pbs-tables + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: core_crypto + pbs_kind: any + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-pbs-benchmark + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-pbs-tables: + name: generate_documentation_svgs/gpu-pbs-tables + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: core_crypto + pbs_kind: any + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-pbs-benchmark + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} diff --git a/Makefile b/Makefile index 815220eaa..a2d4bf1a6 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ GEN_KEY_CACHE_MULTI_BIT_ONLY?=FALSE GEN_KEY_CACHE_COVERAGE_ONLY?=FALSE PARSE_INTEGER_BENCH_CSV_FILE?=tfhe_rs_integer_benches.csv FAST_TESTS?=FALSE -FAST_BENCH?=FALSE +BIT_SIZES_SET?=ALL NIGHTLY_TESTS?=FALSE BENCH_OP_FLAVOR?=DEFAULT BENCH_TYPE?=latency @@ -1353,28 +1353,28 @@ print_doc_bench_parameters: .PHONY: bench_integer # Run benchmarks for unsigned integer bench_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- .PHONY: bench_signed_integer # Run benchmarks for signed integer bench_signed_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- .PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend bench_integer_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- .PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend bench_signed_integer_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1383,7 +1383,7 @@ bench_signed_integer_gpu: install_rs_check_toolchain bench_integer_hpu: install_rs_check_toolchain source ./setup_hpu.sh --config $(HPU_CONFIG); \ export V80_PCIE_DEV=${V80_PCIE_DEV}; \ - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick @@ -1433,7 +1433,7 @@ bench_integer_aes256_gpu: install_rs_check_toolchain .PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters bench_integer_multi_bit: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- @@ -1441,7 +1441,7 @@ bench_integer_multi_bit: install_rs_check_toolchain .PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters bench_signed_integer_multi_bit: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- @@ -1449,7 +1449,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain .PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters bench_integer_multi_bit_gpu: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1457,7 +1457,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain .PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1513,7 +1513,7 @@ bench_pbs: install_rs_check_toolchain .PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend bench_pbs_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench core_crypto-pbs \ --features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark --profile release_lto_off diff --git a/tfhe-benchmark/src/utilities.rs b/tfhe-benchmark/src/utilities.rs index aea999049..fc16e098c 100644 --- a/tfhe-benchmark/src/utilities.rs +++ b/tfhe-benchmark/src/utilities.rs @@ -351,12 +351,33 @@ const BENCH_BIT_SIZES: [usize; 8] = [2, 8, 16, 32, 40, 64, 128, 256]; const BENCH_BIT_SIZES: [usize; 7] = [8, 16, 32, 40, 64, 128, 256]; const HPU_BENCH_BIT_SIZES: [usize; 5] = [8, 16, 32, 64, 128]; const MULTI_BIT_CPU_SIZES: [usize; 5] = [8, 16, 32, 40, 64]; +const BENCH_BIT_SIZES_DOCUMENTATION: [usize; 5] = [8, 16, 32, 64, 128]; + +#[derive(Default)] +pub enum BitSizesSet { + #[default] + Fast, + All, + Documentation, +} + +impl BitSizesSet { + pub fn from_env() -> Result { + let raw_value = env::var("__TFHE_RS_BENCH_BIT_SIZES_SET").unwrap_or("fast".to_string()); + match raw_value.to_lowercase().as_str() { + "fast" => Ok(BitSizesSet::Fast), + "all" => Ok(BitSizesSet::All), + "documentation" => Ok(BitSizesSet::Documentation), + _ => Err(format!("bit sizes set '{raw_value}' is not supported")), + } + } +} /// User configuration in which benchmarks must be run. #[derive(Default)] pub struct EnvConfig { pub is_multi_bit: bool, - pub is_fast_bench: bool, + pub bit_sizes_set: BitSizesSet, } impl EnvConfig { @@ -366,22 +387,21 @@ impl EnvConfig { ParamType::MultiBit | ParamType::MultiBitDocumentation ); - let is_fast_bench = match env::var("__TFHE_RS_FAST_BENCH") { - Ok(val) => val.to_lowercase() == "true", - Err(_) => false, - }; - EnvConfig { is_multi_bit, - is_fast_bench, + bit_sizes_set: BitSizesSet::from_env().unwrap(), } } /// Get precisions values to benchmark. pub fn bit_sizes(&self) -> Vec { - if self.is_fast_bench { - FAST_BENCH_BIT_SIZES.to_vec() - } else if self.is_multi_bit { + let bit_sizes_set = match self.bit_sizes_set { + BitSizesSet::Fast => return FAST_BENCH_BIT_SIZES.to_vec(), + BitSizesSet::All => BENCH_BIT_SIZES.to_vec(), + BitSizesSet::Documentation => return BENCH_BIT_SIZES_DOCUMENTATION.to_vec(), + }; + + if self.is_multi_bit { if cfg!(feature = "gpu") { BENCH_BIT_SIZES.to_vec() } else { @@ -390,7 +410,7 @@ impl EnvConfig { } else if cfg!(feature = "hpu") { HPU_BENCH_BIT_SIZES.to_vec() } else { - BENCH_BIT_SIZES.to_vec() + bit_sizes_set } } }