diff --git a/.github/workflows/benchmark_cpu.yml b/.github/workflows/benchmark_cpu.yml index 3e2d5d37a..01838c043 100644 --- a/.github/workflows/benchmark_cpu.yml +++ b/.github/workflows/benchmark_cpu.yml @@ -34,10 +34,14 @@ on: - smart - unchecked - misc - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -71,7 +75,7 @@ jobs: op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} params_type: ${{ inputs.params_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_cpu_common.yml b/.github/workflows/benchmark_cpu_common.yml index 39ddb6ecb..12c9a97d3 100644 --- a/.github/workflows/benchmark_cpu_common.yml +++ b/.github/workflows/benchmark_cpu_common.yml @@ -16,9 +16,9 @@ on: params_type: type: string default: classical - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast additional_recipe: # Make recipes to run aside the benchmarks. type: string # Use comma separated values to generate an array additional_file_to_parse: # Other files to parse, located under tfhe-benchmark/ directory @@ -54,7 +54,6 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE permissions: {} @@ -69,9 +68,6 @@ jobs: op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} params_type: ${{ steps.set_matrix_args.outputs.params_type }} - env: - INPUTS_COMMAND: ${{ inputs.command }} - INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} steps: - name: Parse user inputs shell: python @@ -162,19 +158,15 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Run benchmarks with AVX512 run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}" + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}" env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_PARAMS_TYPE: ${{ matrix.params_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/benchmark_cpu_weekly.yml b/.github/workflows/benchmark_cpu_weekly.yml index ad44465ae..0a9138e87 100644 --- a/.github/workflows/benchmark_cpu_weekly.yml +++ b/.github/workflows/benchmark_cpu_weekly.yml @@ -30,7 +30,7 @@ jobs: is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }} is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }} op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }} + precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }} steps: - name: Check is weekly bench group 1 id: check_bench_group_1 @@ -52,13 +52,13 @@ jobs: steps.check_bench_group_2.outputs.is_weekly_bench_group_2 == 'true' run: | echo "OP_FLAVOR=default" >> "${GITHUB_ENV}" - echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}" + echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}" - name: Quarterly benchmarks if: steps.check_quarterly_bench.outputs.is_quarterly_bench == 'true' run: | echo "OP_FLAVOR=\"default,unchecked\"" >> "${GITHUB_ENV}" - echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}" + echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}" - name: Set operation flavor output id: set_op_flavor @@ -66,9 +66,9 @@ jobs: echo "op_flavor=${{ env.OP_FLAVOR }}" >> "${GITHUB_OUTPUT}" - name: Set bit precisions output - id: set_all_precisions + id: set_precisions_set run: | # zizmor: ignore[template-injection] this env variable is safe - echo "all_precisions=${{ env.ALL_PRECISIONS }}" >> "${GITHUB_OUTPUT}" + echo "precisions_set=${{ env.PRECISIONS_SET }}" >> "${GITHUB_OUTPUT}" run-benchmarks-integer: name: benchmark_cpu_weekly/run-benchmarks-integer @@ -79,7 +79,7 @@ jobs: with: command: integer,signed_integer, integer_compression op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }} - all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }} + precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_documentation.yml b/.github/workflows/benchmark_documentation.yml index c4da7112f..43c028540 100644 --- a/.github/workflows/benchmark_documentation.yml +++ b/.github/workflows/benchmark_documentation.yml @@ -1,17 +1,212 @@ -name: benchmark_documentation.yml +# Run all benchmarks displayed in the public documentation. +name: benchmark_documentation on: workflow_dispatch: + inputs: + run-cpu-benchmarks: + description: "Run CPU benchmarks" + type: boolean + default: true + run-gpu-benchmarks: + description: "Run GPU benchmarks" + type: boolean + default: true + run-hpu-benchmarks: + description: "Run HPU benchmarks" + type: boolean + default: true + generate-svgs: + description: "Generate SVG tables" + type: boolean + default: true + open-pr: + description: "Open a PR with the benchmark results" + type: boolean + default: false permissions: {} # zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow jobs: - placeholder: - name: benchmark_documentation/placeholder - runs-on: ubuntu-latest + run-benchmarks-cpu-integer: + name: benchmark_documentation/run-benchmarks-cpu-integer + uses: ./.github/workflows/benchmark_cpu_common.yml + if: inputs.run-cpu-benchmarks + with: + command: integer + op_flavor: fast_default + bench_type: both + precisions_set: documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + run-benchmarks-gpu-integer: + name: benchmark_documentation/run-benchmarks-gpu-integer + uses: ./.github/workflows/benchmark_gpu_common.yml + if: inputs.run-gpu-benchmarks + with: + profile: multi-h100-sxm5 + hardware_name: n3-H100-SXM5x8 + command: integer_multi_bit + op_flavor: fast_default + bench_type: both + precisions_set: documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-hpu-integer: + name: benchmark_documentation/run-benchmarks-hpu-integer + uses: ./.github/workflows/benchmark_hpu_common.yml + if: inputs.run-hpu-benchmarks + with: + command: integer + op_flavor: default + bench_type: both + precisions_set: documentation + v80_pcie_dev: 24 + v80_serial_number: XFL12NWY3ZKG + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + + run-benchmarks-cpu-core-crypto: + name: benchmark_documentation/run-benchmarks-cpu-core-crypto + uses: ./.github/workflows/benchmark_cpu_common.yml + if: inputs.run-cpu-benchmarks + with: + command: pbs, ks_pbs + bench_type: latency + params_type: classical_documentation + multi_bit_documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + run-benchmarks-gpu-core-crypto: + name: benchmark_documentation/run-benchmarks-gpu-core-crypto + uses: ./.github/workflows/benchmark_gpu_common.yml + if: inputs.run-gpu-benchmarks + with: + profile: multi-h100-sxm5 + hardware_name: n3-H100-SXM5x8 + command: pbs, ks_pbs + bench_type: latency + params_type: classical_documentation + multi_bit_documentation + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + + generate-svgs-with-benchmarks-run: + name: benchmark-documentation/generate-svgs-with-benchmarks-run + if: ${{ always() && + (inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) && + inputs.generate-svgs }} + needs: [ + run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer, + run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto + ] + uses: ./.github/workflows/generate_svgs.yml + with: + time_span_days: 5 + generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }} + generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }} + generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }} + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + generate-svgs-without-benchmarks-run: + name: benchmark-documentation/generate-svgs-without-benchmarks-run + if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) && + inputs.generate-svgs }} + uses: ./.github/workflows/generate_svgs.yml + with: + time_span_days: 60 + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + open-pr: + name: benchmark-documentation/open-pr + needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ] + if: ${{ always() && inputs.open-pr && + (needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }} + runs-on: ubuntu-latest + permissions: + contents: write # Needed to create a commit + pull-requests: write # Needed to open a pull-request + env: + PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets steps: - - run: | - echo "Hello this is a placeholder workflow" + - name: Checkout tfhe-rs + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + + - name: Download SVG tables + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + path: svg_tables + merge-multiple: 'true' + + - name: Copy SVG tables to documentation location + run: | + cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}" + cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}" + + - name: Get current date + id: get-date + run: | + echo "date=$(date '+%g_%m_%d_%Hh%Mm%Ss')" >> "${GITHUB_OUTPUT}" + + - name: Create pull-request + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + sign-commits: true # Commit will be signed by github-actions bot + add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg + branch: gh-bot/docs/update-svg-tables-${{ steps.get-date.outputs.date }} + commit-message: | + chore(docs): update benchmark results for all backends + + Automated documentation update from tfhe-rs CI pipeline. + title: | + [CI] chore(docs): update benchmark results for all backends + body: | + Documentation update triggered by GitHub workflow. + labels: documentation diff --git a/.github/workflows/benchmark_gpu.yml b/.github/workflows/benchmark_gpu.yml index 4fe370b15..e20fa9e37 100644 --- a/.github/workflows/benchmark_gpu.yml +++ b/.github/workflows/benchmark_gpu.yml @@ -42,10 +42,14 @@ on: - default - fast_default - unchecked - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -108,7 +112,7 @@ jobs: op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} params_type: ${{ inputs.params_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_gpu_4090.yml b/.github/workflows/benchmark_gpu_4090.yml index 1b1479f5c..55f2ea7f3 100644 --- a/.github/workflows/benchmark_gpu_4090.yml +++ b/.github/workflows/benchmark_gpu_4090.yml @@ -11,7 +11,7 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE + BIT_SIZES_SET: FAST on: # Allows you to run this workflow manually from the Actions tab as an alternative. @@ -53,7 +53,6 @@ jobs: echo "BENCH_DATE=$(date --iso-8601=seconds)"; echo "COMMIT_DATE=${COMMIT_DATE}"; echo "COMMIT_HASH=$(git describe --tags --dirty)"; - echo "FAST_BENCH=TRUE"; } >> "${GITHUB_ENV}" env: SHA: ${{ github.sha }} diff --git a/.github/workflows/benchmark_gpu_common.yml b/.github/workflows/benchmark_gpu_common.yml index 6716b87fa..e10bd4346 100644 --- a/.github/workflows/benchmark_gpu_common.yml +++ b/.github/workflows/benchmark_gpu_common.yml @@ -25,9 +25,9 @@ on: params_type: type: string default: multi_bit - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast secrets: REPO_CHECKOUT_TOKEN: required: true @@ -56,8 +56,6 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - FAST_BENCH: TRUE - permissions: {} @@ -68,79 +66,45 @@ jobs: name: benchmark_gpu_common/prepare-matrix runs-on: ubuntu-latest outputs: - command: ${{ steps.set_command.outputs.command }} - op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} - params_type: ${{ steps.set_params_type.outputs.params_type }} - env: - INPUTS_COMMAND: ${{ inputs.command }} - INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} + command: ${{ steps.set_matrix_args.outputs.command }} + op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} + bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} + params_type: ${{ steps.set_matrix_args.outputs.params_type }} steps: - - name: Set single command - if: ${{ !contains(inputs.command, ',')}} - run: | - echo "COMMAND=[\"${INPUTS_COMMAND}\"]" >> "${GITHUB_ENV}" + - name: Parse user inputs + shell: python + run: | # zizmor: ignore[template-injection] these env variables are safe + split_command = "${{ inputs.command }}".replace(" ", "").split(",") + split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",") - - name: Set multiple commands - if: ${{ contains(inputs.command, ',')}} - run: | - # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern. - # shellcheck disable=SC2001 - PARSED_COMMAND=$(echo "${INPUTS_COMMAND}" | sed 's/[[:space:]]*,[[:space:]]*/\", \"/g') - echo "COMMAND=[\"${PARSED_COMMAND}\"]" >> "${GITHUB_ENV}" + if "${{ inputs.bench_type }}" == "both": + bench_type = ["latency", "throughput"] + else: + bench_type = ["${{ inputs.bench_type }}", ] - - name: Set single operations flavor - if: ${{ !contains(inputs.op_flavor, ',')}} - run: | - echo "OP_FLAVOR=[\"${INPUTS_OP_FLAVOR}\"]" >> "${GITHUB_ENV}" + if "+" in "${{ inputs.params_type }}": + split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+") + else: + split_params_type = ["${{ inputs.params_type }}", ] - - name: Set multiple operations flavors - if: ${{ contains(inputs.op_flavor, ',')}} - run: | - # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern. - # shellcheck disable=SC2001 - PARSED_OP_FLAVOR=$(echo "${INPUTS_OP_FLAVOR}" | sed 's/[[:space:]]*,[[:space:]]*/", "/g') - echo "OP_FLAVOR=[\"${PARSED_OP_FLAVOR}\"]" >> "${GITHUB_ENV}" + with open("${{ github.env }}", "a") as f: + for env_name, values_to_join in [ + ("COMMAND", split_command), + ("OP_FLAVOR", split_op_flavor), + ("BENCH_TYPE", bench_type), + ("PARAMS_TYPE", split_params_type), + ]: + f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""") - - name: Set benchmark types - run: | - if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} - - - name: Set parameters types - run: | - if [[ "${INPUTS_PARAMS_TYPE}" == "both" ]]; then - echo "PARAMS_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}" - else - echo "PARAMS_TYPE=[\"${INPUTS_PARAMS_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_PARAMS_TYPE: ${{ inputs.params_type }} - - - name: Set command output - id: set_command - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "command=${{ toJSON(env.COMMAND) }}" >> "${GITHUB_OUTPUT}" - - - name: Set operation flavor output - id: set_op_flavor - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" - - - name: Set parameters types output - id: set_params_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}" >> "${GITHUB_OUTPUT}" + - name: Set martix arguments outputs + id: set_matrix_args + run: | # zizmor: ignore[template-injection] these env variable are safe + { + echo "command=${{ toJSON(env.COMMAND) }}"; + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}"; + echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}"; + echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}"; + } >> "${GITHUB_OUTPUT}" setup-instance: name: benchmark_gpu_common/setup-instance @@ -275,19 +239,15 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Run benchmarks run: | - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_PARAMS_TYPE: ${{ matrix.params_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/benchmark_gpu_weekly.yml b/.github/workflows/benchmark_gpu_weekly.yml index 9cc416c36..9b4f75755 100644 --- a/.github/workflows/benchmark_gpu_weekly.yml +++ b/.github/workflows/benchmark_gpu_weekly.yml @@ -22,7 +22,7 @@ jobs: command: integer_multi_bit op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -43,7 +43,7 @@ jobs: command: integer_compression op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -64,7 +64,7 @@ jobs: command: integer_zk,integer_aes,integer_aes256 op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} @@ -85,7 +85,7 @@ jobs: command: hlapi_noise_squash op_flavor: default bench_type: both - all_precisions: true + precisions_set: fast secrets: BOT_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/benchmark_hpu.yml b/.github/workflows/benchmark_hpu.yml index 630449e9d..d71c02527 100644 --- a/.github/workflows/benchmark_hpu.yml +++ b/.github/workflows/benchmark_hpu.yml @@ -19,10 +19,14 @@ on: options: - default - fast_default - all_precisions: - description: "Run all precisions" - type: boolean - default: false + precisions_set: + description: "Bit precisions set" + type: choice + default: fast + options: + - fast + - all + - documentation bench_type: description: "Benchmarks type" type: choice @@ -50,7 +54,7 @@ jobs: command: ${{ inputs.command }} op_flavor: ${{ inputs.op_flavor }} bench_type: ${{ inputs.bench_type }} - all_precisions: ${{ inputs.all_precisions }} + precisions_set: ${{ inputs.precisions_set }} v80_pcie_dev: ${{ inputs.v80_pcie_dev }} v80_serial_number: ${{ inputs.v80_serial_number }} secrets: diff --git a/.github/workflows/benchmark_hpu_common.yml b/.github/workflows/benchmark_hpu_common.yml index bcc089df4..1b7eea50e 100644 --- a/.github/workflows/benchmark_hpu_common.yml +++ b/.github/workflows/benchmark_hpu_common.yml @@ -13,9 +13,9 @@ on: bench_type: type: string default: latency - all_precisions: - type: boolean - default: false + precisions_set: + type: string + default: fast v80_pcie_dev: type: string default: 24 @@ -48,7 +48,6 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" - FAST_BENCH: TRUE permissions: {} @@ -139,11 +138,6 @@ jobs: with: toolchain: nightly - - name: Should run benchmarks with all precisions - if: inputs.all_precisions - run: | - echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" - - name: Select HPU board run: | echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}" @@ -156,11 +150,12 @@ jobs: run: | echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}" make pull_hpu_files - make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu + make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu env: OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} BENCH_COMMAND: ${{ matrix.command }} + PRECISIONS_SET: ${{ inputs.precisions_set }} - name: Parse results run: | diff --git a/.github/workflows/generate_svg_common.yml b/.github/workflows/generate_svg_common.yml index dc6f2a60a..5900acdca 100644 --- a/.github/workflows/generate_svg_common.yml +++ b/.github/workflows/generate_svg_common.yml @@ -1,17 +1,85 @@ -name: generate_svg_common.yml +name: generate_svg_common on: workflow_call: + inputs: + backend: + type: string + required: true + hardware_name: + type: string + required: true + layer: + type: string + required: true + pbs_kind: # Valid values are 'classical', 'multi_bit' or 'any' + type: string + required: true + grouping_factor: # Valid values are 2, 3, or 4 + type: string + default: 4 + bench_type: # Valid values are 'latency', 'throughput' + type: string + required: true + time_span_days: + type: string + default: 60 + output_filename: + type: string + required: true + secrets: + DATA_EXTRACTOR_DATABASE_USER: + required: true + DATA_EXTRACTOR_DATABASE_HOST: + required: true + DATA_EXTRACTOR_DATABASE_PASSWORD: + required: true permissions: {} # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency jobs: - placeholder: - name: generate_svg_common/placeholder + generate-table: + name: generate_svg_common/generate-table runs-on: ubuntu-latest - steps: - - run: | - echo "Hello this is a placeholder workflow" + - name: Checkout tfhe-rs + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + persist-credentials: 'false' + + - name: Produce table from database + run: | + python3 -m pip install -r ci/data_extractor/requirements.txt + python3 ci/data_extractor/src/data_extractor.py "${OUTPUT_FILENAME}" \ + --generate-svg \ + --branch "${REF_NAME}" \ + --backend "${BACKEND}" \ + --hardware "${HARDWARE_NAME}" \ + --tfhe-rs-layer "${LAYER}" \ + --pbs-kind "${PBS_KIND}" \ + --grouping-factor "${GROUPING_FACTOR}" \ + --bench-type "${BENCH_TYPE}" \ + --time-span-days "${TIME_SPAN}" + env: + OUTPUT_FILENAME: ${{ inputs.output_filename }} + REF_NAME: ${{ github.ref_name }} + BACKEND: ${{ inputs.backend }} + HARDWARE_NAME: ${{ inputs.hardware_name }} + LAYER: ${{ inputs.layer }} + PBS_KIND: ${{ inputs.pbs_kind }} + GROUPING_FACTOR: ${{ inputs.grouping_factor }} + BENCH_TYPE: ${{ inputs.bench_type }} + TIME_SPAN: ${{ inputs.time_span_days }} + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + - name: Upload tables + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 + with: + name: ${{ github.sha }}_${{ inputs.backend }}_${{ inputs.layer }}_${{ inputs.pbs_kind }}_${{ inputs.bench_type }}_tables + # This will upload all the file generated + path: ${{ inputs.output_filename }}*.svg + retention-days: 60 diff --git a/.github/workflows/generate_svgs.yml b/.github/workflows/generate_svgs.yml index f84362dd5..40bb1ef1a 100644 --- a/.github/workflows/generate_svgs.yml +++ b/.github/workflows/generate_svgs.yml @@ -1,17 +1,178 @@ -name: generate_svgs.yml +# Generate benchmark SVGs for public documentation +name: generate_documentation_svgs on: workflow_call: + inputs: + time_span_days: + type: string + required: true + generate-cpu-svgs: + type: boolean + default: true + generate-gpu-svgs: + type: boolean + default: true + generate-hpu-svgs: + type: boolean + default: true + secrets: + DATA_EXTRACTOR_DATABASE_USER: + required: true + DATA_EXTRACTOR_DATABASE_HOST: + required: true + DATA_EXTRACTOR_DATABASE_PASSWORD: + required: true permissions: {} # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency jobs: - placeholder: - name: generate_svgs/placeholder - runs-on: ubuntu-latest + # ----------------------------------------------------------- + # Integer benchmarks tables + # ----------------------------------------------------------- - steps: - - run: | - echo "Hello this is a placeholder workflow" + cpu-integer-latency-table: + name: generate_documentation_svgs/cpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: integer + pbs_kind: classical + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-integer-benchmark-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + cpu-integer-throughput-table: + name: generate_documentation_svgs/cpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: integer + pbs_kind: classical + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-integer-benchmark-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-integer-latency-table: + name: generate_documentation_svgs/gpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: integer + pbs_kind: multi_bit + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-integer-throughput-table: + name: generate_documentation_svgs/gpu-integer-throughput-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: integer + pbs_kind: multi_bit + grouping_factor: 4 + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + hpu-integer-latency-table: + name: generate_documentation_svgs/hpu-integer-latency-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-hpu-svgs + with: + backend: hpu + hardware_name: hpu_x1 + layer: integer + pbs_kind: classical + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-latency + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + hpu-integer-throughput-table: + name: generate_documentation_svgs/hpu-integer-throughput-table + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-hpu-svgs + with: + backend: hpu + hardware_name: hpu_x1 + layer: integer + pbs_kind: classical + bench_type: throughput + time_span_days: ${{ inputs.time_span_days }} + output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-throughput + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + # ----------------------------------------------------------- + # PBS benchmarks tables + # ----------------------------------------------------------- + + cpu-pbs-tables: + name: generate_documentation_svgs/cpu-pbs-tables + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-cpu-svgs + with: + backend: cpu + hardware_name: hpc7a.96xlarge + layer: core_crypto + pbs_kind: any + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: cpu-pbs-benchmark + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} + + gpu-pbs-tables: + name: generate_documentation_svgs/gpu-pbs-tables + uses: ./.github/workflows/generate_svg_common.yml + if: inputs.generate-gpu-svgs + with: + backend: gpu + hardware_name: n3-H100-SXM5x8 + layer: core_crypto + pbs_kind: any + grouping_factor: 4 + bench_type: latency + time_span_days: ${{ inputs.time_span_days }} + output_filename: gpu-pbs-benchmark + secrets: + DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }} + DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }} + DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }} diff --git a/Makefile b/Makefile index 815220eaa..a2d4bf1a6 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ GEN_KEY_CACHE_MULTI_BIT_ONLY?=FALSE GEN_KEY_CACHE_COVERAGE_ONLY?=FALSE PARSE_INTEGER_BENCH_CSV_FILE?=tfhe_rs_integer_benches.csv FAST_TESTS?=FALSE -FAST_BENCH?=FALSE +BIT_SIZES_SET?=ALL NIGHTLY_TESTS?=FALSE BENCH_OP_FLAVOR?=DEFAULT BENCH_TYPE?=latency @@ -1353,28 +1353,28 @@ print_doc_bench_parameters: .PHONY: bench_integer # Run benchmarks for unsigned integer bench_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- .PHONY: bench_signed_integer # Run benchmarks for signed integer bench_signed_integer: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- .PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend bench_integer_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- .PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend bench_signed_integer_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1383,7 +1383,7 @@ bench_signed_integer_gpu: install_rs_check_toolchain bench_integer_hpu: install_rs_check_toolchain source ./setup_hpu.sh --config $(HPU_CONFIG); \ export V80_PCIE_DEV=${V80_PCIE_DEV}; \ - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick @@ -1433,7 +1433,7 @@ bench_integer_aes256_gpu: install_rs_check_toolchain .PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters bench_integer_multi_bit: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- @@ -1441,7 +1441,7 @@ bench_integer_multi_bit: install_rs_check_toolchain .PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters bench_signed_integer_multi_bit: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark -- @@ -1449,7 +1449,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain .PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters bench_integer_multi_bit_gpu: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1457,7 +1457,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain .PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \ - __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench integer-signed \ --features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off -- @@ -1513,7 +1513,7 @@ bench_pbs: install_rs_check_toolchain .PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend bench_pbs_gpu: install_rs_check_toolchain - RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ + RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \ cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \ --bench core_crypto-pbs \ --features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark --profile release_lto_off diff --git a/tfhe-benchmark/src/utilities.rs b/tfhe-benchmark/src/utilities.rs index aea999049..fc16e098c 100644 --- a/tfhe-benchmark/src/utilities.rs +++ b/tfhe-benchmark/src/utilities.rs @@ -351,12 +351,33 @@ const BENCH_BIT_SIZES: [usize; 8] = [2, 8, 16, 32, 40, 64, 128, 256]; const BENCH_BIT_SIZES: [usize; 7] = [8, 16, 32, 40, 64, 128, 256]; const HPU_BENCH_BIT_SIZES: [usize; 5] = [8, 16, 32, 64, 128]; const MULTI_BIT_CPU_SIZES: [usize; 5] = [8, 16, 32, 40, 64]; +const BENCH_BIT_SIZES_DOCUMENTATION: [usize; 5] = [8, 16, 32, 64, 128]; + +#[derive(Default)] +pub enum BitSizesSet { + #[default] + Fast, + All, + Documentation, +} + +impl BitSizesSet { + pub fn from_env() -> Result { + let raw_value = env::var("__TFHE_RS_BENCH_BIT_SIZES_SET").unwrap_or("fast".to_string()); + match raw_value.to_lowercase().as_str() { + "fast" => Ok(BitSizesSet::Fast), + "all" => Ok(BitSizesSet::All), + "documentation" => Ok(BitSizesSet::Documentation), + _ => Err(format!("bit sizes set '{raw_value}' is not supported")), + } + } +} /// User configuration in which benchmarks must be run. #[derive(Default)] pub struct EnvConfig { pub is_multi_bit: bool, - pub is_fast_bench: bool, + pub bit_sizes_set: BitSizesSet, } impl EnvConfig { @@ -366,22 +387,21 @@ impl EnvConfig { ParamType::MultiBit | ParamType::MultiBitDocumentation ); - let is_fast_bench = match env::var("__TFHE_RS_FAST_BENCH") { - Ok(val) => val.to_lowercase() == "true", - Err(_) => false, - }; - EnvConfig { is_multi_bit, - is_fast_bench, + bit_sizes_set: BitSizesSet::from_env().unwrap(), } } /// Get precisions values to benchmark. pub fn bit_sizes(&self) -> Vec { - if self.is_fast_bench { - FAST_BENCH_BIT_SIZES.to_vec() - } else if self.is_multi_bit { + let bit_sizes_set = match self.bit_sizes_set { + BitSizesSet::Fast => return FAST_BENCH_BIT_SIZES.to_vec(), + BitSizesSet::All => BENCH_BIT_SIZES.to_vec(), + BitSizesSet::Documentation => return BENCH_BIT_SIZES_DOCUMENTATION.to_vec(), + }; + + if self.is_multi_bit { if cfg!(feature = "gpu") { BENCH_BIT_SIZES.to_vec() } else { @@ -390,7 +410,7 @@ impl EnvConfig { } else if cfg!(feature = "hpu") { HPU_BENCH_BIT_SIZES.to_vec() } else { - BENCH_BIT_SIZES.to_vec() + bit_sizes_set } } }