chore(ci): add workflow to update documentation benchmark tables

This new workflow can trigger all the required benchmarks needed
to populate benchmarks tables in documentation.
It also can generate SVG tables and store them as artifacts.
Optionally, it can open a pull-request to update the current
tables in documentation.

.github/workflows/benchmark_cpu.yml

@@ -34,10 +34,14 @@ on:
           - smart
           - unchecked
           - misc
-      all_precisions:
+      precisions_set:
-        description: "Run all precisions"
+        description: "Bit precisions set"
-        type: boolean
+        type: choice
-        default: false
+        default: fast
+        options:
+          - fast
+          - all
+          - documentation
       bench_type:
         description: "Benchmarks type"
         type: choice
@@ -71,7 +75,7 @@ jobs:
       op_flavor: ${{ inputs.op_flavor }}
       bench_type: ${{ inputs.bench_type }}
       params_type: ${{ inputs.params_type }}
-      all_precisions: ${{ inputs.all_precisions }}
+      precisions_set: ${{ inputs.precisions_set }}
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_cpu_common.yml

@@ -16,9 +16,9 @@ on:
       params_type:
         type: string
         default: classical
-      all_precisions:
+      precisions_set:
-        type: boolean
+        type: string
-        default: false
+        default: fast
       additional_recipe: # Make recipes to run aside the benchmarks.
         type: string # Use comma separated values to generate an array
       additional_file_to_parse: # Other files to parse, located under tfhe-benchmark/ directory
@@ -54,7 +54,6 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-  FAST_BENCH: TRUE
 
 permissions: {}
 
@@ -69,9 +68,6 @@ jobs:
       op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
       bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
       params_type: ${{ steps.set_matrix_args.outputs.params_type }}
-    env:
-      INPUTS_COMMAND: ${{ inputs.command }}
-      INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
     steps:
       - name: Parse user inputs
         shell: python
@@ -162,19 +158,15 @@ jobs:
         with:
           toolchain: nightly
 
-      - name: Should run benchmarks with all precisions
-        if: inputs.all_precisions
-        run: |
-          echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
-
       - name: Run benchmarks with AVX512
         run: |
-          make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
+          make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
         env:
           OP_FLAVOR: ${{ matrix.op_flavor }}
           BENCH_TYPE: ${{ matrix.bench_type }}
           BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
           BENCH_COMMAND: ${{ matrix.command }}
+          PRECISIONS_SET: ${{ inputs.precisions_set }}
 
       - name: Parse results
         run: |

.github/workflows/benchmark_cpu_weekly.yml

@@ -30,7 +30,7 @@ jobs:
       is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }}
       is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }}
       op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
-      all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }}
+      precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }}
     steps:
       - name: Check is weekly bench group 1
         id: check_bench_group_1
@@ -52,13 +52,13 @@ jobs:
           steps.check_bench_group_2.outputs.is_weekly_bench_group_2 == 'true'
         run: |
           echo "OP_FLAVOR=default" >> "${GITHUB_ENV}"
-          echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}"
+          echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}"
 
       - name: Quarterly benchmarks
         if: steps.check_quarterly_bench.outputs.is_quarterly_bench == 'true'
         run: |
           echo "OP_FLAVOR=\"default,unchecked\"" >> "${GITHUB_ENV}"
-          echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}"
+          echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}"
 
       - name: Set operation flavor output
         id: set_op_flavor
@@ -66,9 +66,9 @@ jobs:
           echo "op_flavor=${{ env.OP_FLAVOR }}" >> "${GITHUB_OUTPUT}"
 
       - name: Set bit precisions output
-        id: set_all_precisions
+        id: set_precisions_set
         run: | # zizmor: ignore[template-injection] this env variable is safe
-          echo "all_precisions=${{ env.ALL_PRECISIONS }}" >> "${GITHUB_OUTPUT}"
+          echo "precisions_set=${{ env.PRECISIONS_SET }}" >> "${GITHUB_OUTPUT}"
 
   run-benchmarks-integer:
     name: benchmark_cpu_weekly/run-benchmarks-integer
@@ -79,7 +79,7 @@ jobs:
     with:
       command: integer,signed_integer, integer_compression
       op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }}
-      all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }}
+      precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }}
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_documentation.yml

@@ -1,17 +1,212 @@
-name: benchmark_documentation.yml
+# Run all benchmarks displayed in the public documentation.
+name: benchmark_documentation
 
 on:
   workflow_dispatch:
+    inputs:
+      run-cpu-benchmarks:
+        description: "Run CPU benchmarks"
+        type: boolean
+        default: true
+      run-gpu-benchmarks:
+        description: "Run GPU benchmarks"
+        type: boolean
+        default: true
+      run-hpu-benchmarks:
+        description: "Run HPU benchmarks"
+        type: boolean
+        default: true
+      generate-svgs:
+        description: "Generate SVG tables"
+        type: boolean
+        default: true
+      open-pr:
+        description: "Open a PR with the benchmark results"
+        type: boolean
+        default: false
 
 permissions: {}
 
 # zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow
 
 jobs:
-  placeholder:
+  run-benchmarks-cpu-integer:
-    name: benchmark_documentation/placeholder
+    name: benchmark_documentation/run-benchmarks-cpu-integer
-    runs-on: ubuntu-latest
+    uses: ./.github/workflows/benchmark_cpu_common.yml
+    if: inputs.run-cpu-benchmarks
+    with:
+      command: integer
+      op_flavor: fast_default
+      bench_type: both
+      precisions_set: documentation
+    secrets:
+      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
+      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+      JOB_SECRET: ${{ secrets.JOB_SECRET }}
+      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
+      SLAB_URL: ${{ secrets.SLAB_URL }}
+      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
 
+  run-benchmarks-gpu-integer:
+    name: benchmark_documentation/run-benchmarks-gpu-integer
+    uses: ./.github/workflows/benchmark_gpu_common.yml
+    if: inputs.run-gpu-benchmarks
+    with:
+      profile: multi-h100-sxm5
+      hardware_name: n3-H100-SXM5x8
+      command: integer_multi_bit
+      op_flavor: fast_default
+      bench_type: both
+      precisions_set: documentation
+    secrets:
+      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
+      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+      JOB_SECRET: ${{ secrets.JOB_SECRET }}
+      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
+      SLAB_URL: ${{ secrets.SLAB_URL }}
+      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
+
+  run-benchmarks-hpu-integer:
+    name: benchmark_documentation/run-benchmarks-hpu-integer
+    uses: ./.github/workflows/benchmark_hpu_common.yml
+    if: inputs.run-hpu-benchmarks
+    with:
+      command: integer
+      op_flavor: default
+      bench_type: both
+      precisions_set: documentation
+      v80_pcie_dev: 24
+      v80_serial_number: XFL12NWY3ZKG
+    secrets:
+      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
+      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+      JOB_SECRET: ${{ secrets.JOB_SECRET }}
+      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
+      SLAB_URL: ${{ secrets.SLAB_URL }}
+      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
+      SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+
+  run-benchmarks-cpu-core-crypto:
+    name: benchmark_documentation/run-benchmarks-cpu-core-crypto
+    uses: ./.github/workflows/benchmark_cpu_common.yml
+    if: inputs.run-cpu-benchmarks
+    with:
+      command: pbs, ks_pbs
+      bench_type: latency
+      params_type: classical_documentation + multi_bit_documentation
+    secrets:
+      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
+      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+      JOB_SECRET: ${{ secrets.JOB_SECRET }}
+      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
+      SLAB_URL: ${{ secrets.SLAB_URL }}
+      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
+
+  run-benchmarks-gpu-core-crypto:
+    name: benchmark_documentation/run-benchmarks-gpu-core-crypto
+    uses: ./.github/workflows/benchmark_gpu_common.yml
+    if: inputs.run-gpu-benchmarks
+    with:
+      profile: multi-h100-sxm5
+      hardware_name: n3-H100-SXM5x8
+      command: pbs, ks_pbs
+      bench_type: latency
+      params_type: classical_documentation + multi_bit_documentation
+    secrets:
+      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
+      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+      JOB_SECRET: ${{ secrets.JOB_SECRET }}
+      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
+      SLAB_URL: ${{ secrets.SLAB_URL }}
+      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
+
+  generate-svgs-with-benchmarks-run:
+    name: benchmark-documentation/generate-svgs-with-benchmarks-run
+    if: ${{ always() &&
+      (inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) &&
+      inputs.generate-svgs }}
+    needs: [
+      run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
+      run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
+    ]
+    uses: ./.github/workflows/generate_svgs.yml
+    with:
+      time_span_days: 5
+      generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }}
+      generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }}
+      generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }}
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  generate-svgs-without-benchmarks-run:
+    name: benchmark-documentation/generate-svgs-without-benchmarks-run
+    if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) &&
+      inputs.generate-svgs }}
+    uses: ./.github/workflows/generate_svgs.yml
+    with:
+      time_span_days: 60
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  open-pr:
+    name: benchmark-documentation/open-pr
+    needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ]
+    if: ${{ always() && inputs.open-pr &&
+      (needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # Needed to create a commit
+      pull-requests: write # Needed to open a pull-request
+    env:
+      PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets
     steps:
-      - run: |
+      - name: Checkout tfhe-rs
-          echo "Hello this is a placeholder workflow"
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+
+      - name: Download SVG tables
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          path: svg_tables
+          merge-multiple: 'true'
+
+      - name: Copy SVG tables to documentation location
+        run: |
+          cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}"
+          cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}"
+
+      - name: Get current date
+        id: get-date
+        run: |
+          echo "date=$(date '+%g_%m_%d_%Hh%Mm%Ss')" >> "${GITHUB_OUTPUT}"
+
+      - name: Create pull-request
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
+        with:
+          sign-commits: true # Commit will be signed by github-actions bot
+          add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg
+          branch: gh-bot/docs/update-svg-tables-${{ steps.get-date.outputs.date }}
+          commit-message: |
+            chore(docs): update benchmark results for all backends
+
+            Automated documentation update from tfhe-rs CI pipeline.
+          title: |
+            [CI] chore(docs): update benchmark results for all backends
+          body: |
+            Documentation update triggered by GitHub workflow.
+          labels: documentation

.github/workflows/benchmark_gpu.yml

@@ -42,10 +42,14 @@ on:
           - default
           - fast_default
           - unchecked
-      all_precisions:
+      precisions_set:
-        description: "Run all precisions"
+        description: "Bit precisions set"
-        type: boolean
+        type: choice
-        default: false
+        default: fast
+        options:
+          - fast
+          - all
+          - documentation
       bench_type:
         description: "Benchmarks type"
         type: choice
@@ -108,7 +112,7 @@ jobs:
       op_flavor: ${{ inputs.op_flavor }}
       bench_type: ${{ inputs.bench_type }}
       params_type: ${{ inputs.params_type }}
-      all_precisions: ${{ inputs.all_precisions }}
+      precisions_set: ${{ inputs.precisions_set }}
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_gpu_4090.yml

@@ -11,7 +11,7 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-  FAST_BENCH: TRUE
+  BIT_SIZES_SET: FAST
 
 on:
   # Allows you to run this workflow manually from the Actions tab as an alternative.
@@ -53,7 +53,6 @@ jobs:
             echo "BENCH_DATE=$(date --iso-8601=seconds)";
             echo "COMMIT_DATE=${COMMIT_DATE}";
             echo "COMMIT_HASH=$(git describe --tags --dirty)";
-            echo "FAST_BENCH=TRUE";
           } >> "${GITHUB_ENV}"
         env:
           SHA: ${{ github.sha }}

.github/workflows/benchmark_gpu_common.yml

@@ -25,9 +25,9 @@ on:
       params_type:
         type: string
         default: multi_bit
-      all_precisions:
+      precisions_set:
-        type: boolean
+        type: string
-        default: false
+        default: fast
     secrets:
       REPO_CHECKOUT_TOKEN:
         required: true
@@ -56,8 +56,6 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-  FAST_BENCH: TRUE
-
 
 permissions: {}
 
@@ -68,79 +66,45 @@ jobs:
     name: benchmark_gpu_common/prepare-matrix
     runs-on: ubuntu-latest
     outputs:
-      command: ${{ steps.set_command.outputs.command }}
+      command: ${{ steps.set_matrix_args.outputs.command }}
-      op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
+      op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
-      bench_type: ${{ steps.set_bench_type.outputs.bench_type }}
+      bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
-      params_type: ${{ steps.set_params_type.outputs.params_type }}
+      params_type: ${{ steps.set_matrix_args.outputs.params_type }}
-    env:
-      INPUTS_COMMAND: ${{ inputs.command }}
-      INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
     steps:
-      - name: Set single command
+      - name: Parse user inputs
-        if: ${{ !contains(inputs.command, ',')}}
+        shell: python
-        run: |
+        run: | # zizmor: ignore[template-injection] these env variables are safe
-          echo "COMMAND=[\"${INPUTS_COMMAND}\"]" >> "${GITHUB_ENV}"
+          split_command = "${{ inputs.command }}".replace(" ", "").split(",")
+          split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",")
 
-      - name: Set multiple commands
+          if "${{ inputs.bench_type }}" == "both":
-        if: ${{ contains(inputs.command, ',')}}
+            bench_type = ["latency", "throughput"]
-        run: |
+          else:
-          # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
+            bench_type = ["${{ inputs.bench_type }}", ]
-          # shellcheck disable=SC2001
-          PARSED_COMMAND=$(echo "${INPUTS_COMMAND}" | sed 's/[[:space:]]*,[[:space:]]*/\", \"/g')
-          echo "COMMAND=[\"${PARSED_COMMAND}\"]" >> "${GITHUB_ENV}"
 
-      - name: Set single operations flavor
+          if "+" in "${{ inputs.params_type }}":
-        if: ${{ !contains(inputs.op_flavor, ',')}}
+            split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+")
-        run: |
+          else:
-          echo "OP_FLAVOR=[\"${INPUTS_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
+            split_params_type = ["${{ inputs.params_type }}", ]
 
-      - name: Set multiple operations flavors
+          with open("${{ github.env }}", "a") as f:
-        if: ${{ contains(inputs.op_flavor, ',')}}
+            for env_name, values_to_join in [
-        run: |
+              ("COMMAND", split_command),
-          # Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
+              ("OP_FLAVOR", split_op_flavor),
-          # shellcheck disable=SC2001
+              ("BENCH_TYPE", bench_type),
-          PARSED_OP_FLAVOR=$(echo "${INPUTS_OP_FLAVOR}" | sed 's/[[:space:]]*,[[:space:]]*/", "/g')
+              ("PARAMS_TYPE", split_params_type),
-          echo "OP_FLAVOR=[\"${PARSED_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
+            ]:
+              f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""")
 
-      - name: Set benchmark types
+      - name: Set martix arguments outputs
-        run: |
+        id: set_matrix_args
-          if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then
+        run: | # zizmor: ignore[template-injection] these env variable are safe
-            echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}"
+          {
-          else
+            echo "command=${{ toJSON(env.COMMAND) }}";
-            echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}"
+            echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}";
-          fi
+            echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}";
-        env:
+            echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}";
-          INPUTS_BENCH_TYPE: ${{ inputs.bench_type }}
+          } >> "${GITHUB_OUTPUT}"
-
-      - name: Set parameters types
-        run: |
-          if [[ "${INPUTS_PARAMS_TYPE}" == "both" ]]; then
-            echo "PARAMS_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}"
-          else
-            echo "PARAMS_TYPE=[\"${INPUTS_PARAMS_TYPE}\"]" >> "${GITHUB_ENV}"
-          fi
-        env:
-          INPUTS_PARAMS_TYPE: ${{ inputs.params_type }}
-
-      - name: Set command output
-        id: set_command
-        run: | # zizmor: ignore[template-injection] this env variable is safe
-          echo "command=${{ toJSON(env.COMMAND) }}" >> "${GITHUB_OUTPUT}"
-
-      - name: Set operation flavor output
-        id: set_op_flavor
-        run: | # zizmor: ignore[template-injection] this env variable is safe
-          echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}"
-
-      - name: Set benchmark types output
-        id: set_bench_type
-        run: | # zizmor: ignore[template-injection] this env variable is safe
-          echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}"
-
-      - name: Set parameters types output
-        id: set_params_type
-        run: | # zizmor: ignore[template-injection] this env variable is safe
-          echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}" >> "${GITHUB_OUTPUT}"
 
   setup-instance:
     name: benchmark_gpu_common/setup-instance
@@ -275,19 +239,15 @@ jobs:
         with:
           toolchain: nightly
 
-      - name: Should run benchmarks with all precisions
-        if: inputs.all_precisions
-        run: |
-          echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
-
       - name: Run benchmarks
         run: |
-          make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
+          make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
         env:
           OP_FLAVOR: ${{ matrix.op_flavor }}
           BENCH_TYPE: ${{ matrix.bench_type }}
           BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
           BENCH_COMMAND: ${{ matrix.command }}
+          PRECISIONS_SET: ${{ inputs.precisions_set }}
 
       - name: Parse results
         run: |

.github/workflows/benchmark_gpu_weekly.yml

@@ -22,7 +22,7 @@ jobs:
       command: integer_multi_bit
       op_flavor: default
       bench_type: both
-      all_precisions: true
+      precisions_set: fast
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
@@ -43,7 +43,7 @@ jobs:
       command: integer_compression
       op_flavor: default
       bench_type: both
-      all_precisions: true
+      precisions_set: fast
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
@@ -64,7 +64,7 @@ jobs:
       command: integer_zk,integer_aes,integer_aes256
       op_flavor: default
       bench_type: both
-      all_precisions: true
+      precisions_set: fast
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
@@ -85,7 +85,7 @@ jobs:
       command: hlapi_noise_squash
       op_flavor: default
       bench_type: both
-      all_precisions: true
+      precisions_set: fast
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_hpu.yml

@@ -19,10 +19,14 @@ on:
         options:
           - default
           - fast_default
-      all_precisions:
+      precisions_set:
-        description: "Run all precisions"
+        description: "Bit precisions set"
-        type: boolean
+        type: choice
-        default: false
+        default: fast
+        options:
+          - fast
+          - all
+          - documentation
       bench_type:
         description: "Benchmarks type"
         type: choice
@@ -50,7 +54,7 @@ jobs:
       command: ${{ inputs.command }}
       op_flavor: ${{ inputs.op_flavor }}
       bench_type: ${{ inputs.bench_type }}
-      all_precisions: ${{ inputs.all_precisions }}
+      precisions_set: ${{ inputs.precisions_set }}
       v80_pcie_dev: ${{ inputs.v80_pcie_dev }}
       v80_serial_number: ${{ inputs.v80_serial_number }}
     secrets:

.github/workflows/benchmark_hpu_common.yml

@@ -13,9 +13,9 @@ on:
       bench_type:
         type: string
         default: latency
-      all_precisions:
+      precisions_set:
-        type: boolean
+        type: string
-        default: false
+        default: fast
       v80_pcie_dev:
         type: string
         default: 24
@@ -48,7 +48,6 @@ env:
   ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
   RUST_BACKTRACE: "full"
   RUST_MIN_STACK: "8388608"
-  FAST_BENCH: TRUE
 
 permissions: {}
 
@@ -139,11 +138,6 @@ jobs:
         with:
           toolchain: nightly
 
-      - name: Should run benchmarks with all precisions
-        if: inputs.all_precisions
-        run: |
-          echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
-
       - name: Select HPU board
         run: |
           echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}"
@@ -156,11 +150,12 @@ jobs:
         run: |
           echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}"
           make pull_hpu_files
-          make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
+          make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
         env:
           OP_FLAVOR: ${{ matrix.op_flavor }}
           BENCH_TYPE: ${{ matrix.bench_type }}
           BENCH_COMMAND: ${{ matrix.command }}
+          PRECISIONS_SET: ${{ inputs.precisions_set }}
 
       - name: Parse results
         run: |

.github/workflows/generate_svg_common.yml

@@ -1,17 +1,85 @@
-name: generate_svg_common.yml
+name: generate_svg_common
 
 on:
   workflow_call:
+    inputs:
+      backend:
+        type: string
+        required: true
+      hardware_name:
+        type: string
+        required: true
+      layer:
+        type: string
+        required: true
+      pbs_kind: # Valid values are 'classical', 'multi_bit' or 'any'
+        type: string
+        required: true
+      grouping_factor: # Valid values are 2, 3, or 4
+        type: string
+        default: 4
+      bench_type: # Valid values are 'latency', 'throughput'
+        type: string
+        required: true
+      time_span_days:
+        type: string
+        default: 60
+      output_filename:
+        type: string
+        required: true
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER:
+        required: true
+      DATA_EXTRACTOR_DATABASE_HOST:
+        required: true
+      DATA_EXTRACTOR_DATABASE_PASSWORD:
+        required: true
 
 permissions: {}
 
 # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency
 
 jobs:
-  placeholder:
+  generate-table:
-    name: generate_svg_common/placeholder
+    name: generate_svg_common/generate-table
     runs-on: ubuntu-latest
-
     steps:
-      - run: |
+      - name: Checkout tfhe-rs
-          echo "Hello this is a placeholder workflow"
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+        with:
+          persist-credentials: 'false'
+
+      - name: Produce table from database
+        run: |
+          python3 -m pip install -r ci/data_extractor/requirements.txt
+          python3 ci/data_extractor/src/data_extractor.py "${OUTPUT_FILENAME}" \
+          --generate-svg \
+          --branch "${REF_NAME}" \
+          --backend "${BACKEND}" \
+          --hardware "${HARDWARE_NAME}" \
+          --tfhe-rs-layer "${LAYER}" \
+          --pbs-kind "${PBS_KIND}" \
+          --grouping-factor "${GROUPING_FACTOR}" \
+          --bench-type "${BENCH_TYPE}" \
+          --time-span-days "${TIME_SPAN}"
+        env:
+          OUTPUT_FILENAME: ${{ inputs.output_filename }}
+          REF_NAME: ${{ github.ref_name }}
+          BACKEND: ${{ inputs.backend }}
+          HARDWARE_NAME: ${{ inputs.hardware_name }}
+          LAYER: ${{ inputs.layer }}
+          PBS_KIND: ${{ inputs.pbs_kind }}
+          GROUPING_FACTOR: ${{ inputs.grouping_factor }}
+          BENCH_TYPE: ${{ inputs.bench_type }}
+          TIME_SPAN: ${{ inputs.time_span_days }}
+          DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+          DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+          DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+      - name: Upload tables
+        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
+        with:
+          name: ${{ github.sha }}_${{ inputs.backend }}_${{ inputs.layer }}_${{ inputs.pbs_kind }}_${{ inputs.bench_type }}_tables
+          # This will upload all the file generated
+          path: ${{ inputs.output_filename }}*.svg
+          retention-days: 60

.github/workflows/generate_svgs.yml

@@ -1,17 +1,178 @@
-name: generate_svgs.yml
+# Generate benchmark SVGs for public documentation
+name: generate_documentation_svgs
 
 on:
   workflow_call:
+    inputs:
+      time_span_days:
+        type: string
+        required: true
+      generate-cpu-svgs:
+        type: boolean
+        default: true
+      generate-gpu-svgs:
+        type: boolean
+        default: true
+      generate-hpu-svgs:
+        type: boolean
+        default: true
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER:
+        required: true
+      DATA_EXTRACTOR_DATABASE_HOST:
+        required: true
+      DATA_EXTRACTOR_DATABASE_PASSWORD:
+        required: true
 
 permissions: {}
 
 # zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency
 
 jobs:
-  placeholder:
+  # -----------------------------------------------------------
-    name: generate_svgs/placeholder
+  # Integer benchmarks tables
-    runs-on: ubuntu-latest
+  # -----------------------------------------------------------
 
-    steps:
+  cpu-integer-latency-table:
-      - run: |
+    name: generate_documentation_svgs/cpu-integer-latency-table
-          echo "Hello this is a placeholder workflow"
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-cpu-svgs
+    with:
+      backend: cpu
+      hardware_name: hpc7a.96xlarge
+      layer: integer
+      pbs_kind: classical
+      bench_type: latency
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: cpu-integer-benchmark-tuniform-2m128-latency
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  cpu-integer-throughput-table:
+    name: generate_documentation_svgs/cpu-integer-latency-table
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-cpu-svgs
+    with:
+      backend: cpu
+      hardware_name: hpc7a.96xlarge
+      layer: integer
+      pbs_kind: classical
+      bench_type: throughput
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: cpu-integer-benchmark-tuniform-2m128-throughput
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  gpu-integer-latency-table:
+    name: generate_documentation_svgs/gpu-integer-latency-table
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-gpu-svgs
+    with:
+      backend: gpu
+      hardware_name: n3-H100-SXM5x8
+      layer: integer
+      pbs_kind: multi_bit
+      grouping_factor: 4
+      bench_type: latency
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-latency
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  gpu-integer-throughput-table:
+    name: generate_documentation_svgs/gpu-integer-throughput-table
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-gpu-svgs
+    with:
+      backend: gpu
+      hardware_name: n3-H100-SXM5x8
+      layer: integer
+      pbs_kind: multi_bit
+      grouping_factor: 4
+      bench_type: throughput
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-throughput
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  hpu-integer-latency-table:
+    name: generate_documentation_svgs/hpu-integer-latency-table
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-hpu-svgs
+    with:
+      backend: hpu
+      hardware_name: hpu_x1
+      layer: integer
+      pbs_kind: classical
+      bench_type: latency
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-latency
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  hpu-integer-throughput-table:
+    name: generate_documentation_svgs/hpu-integer-throughput-table
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-hpu-svgs
+    with:
+      backend: hpu
+      hardware_name: hpu_x1
+      layer: integer
+      pbs_kind: classical
+      bench_type: throughput
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-throughput
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  # -----------------------------------------------------------
+  # PBS benchmarks tables
+  # -----------------------------------------------------------
+
+  cpu-pbs-tables:
+    name: generate_documentation_svgs/cpu-pbs-tables
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-cpu-svgs
+    with:
+      backend: cpu
+      hardware_name: hpc7a.96xlarge
+      layer: core_crypto
+      pbs_kind: any
+      grouping_factor: 4
+      bench_type: latency
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: cpu-pbs-benchmark
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
+
+  gpu-pbs-tables:
+    name: generate_documentation_svgs/gpu-pbs-tables
+    uses: ./.github/workflows/generate_svg_common.yml
+    if: inputs.generate-gpu-svgs
+    with:
+      backend: gpu
+      hardware_name: n3-H100-SXM5x8
+      layer: core_crypto
+      pbs_kind: any
+      grouping_factor: 4
+      bench_type: latency
+      time_span_days: ${{ inputs.time_span_days }}
+      output_filename: gpu-pbs-benchmark
+    secrets:
+      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
+      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
+      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}

Makefile

@@ -15,7 +15,7 @@ GEN_KEY_CACHE_MULTI_BIT_ONLY?=FALSE
 GEN_KEY_CACHE_COVERAGE_ONLY?=FALSE
 PARSE_INTEGER_BENCH_CSV_FILE?=tfhe_rs_integer_benches.csv
 FAST_TESTS?=FALSE
-FAST_BENCH?=FALSE
+BIT_SIZES_SET?=ALL
 NIGHTLY_TESTS?=FALSE
 BENCH_OP_FLAVOR?=DEFAULT
 BENCH_TYPE?=latency
@@ -1353,28 +1353,28 @@ print_doc_bench_parameters:
 
 .PHONY: bench_integer # Run benchmarks for unsigned integer
 bench_integer: install_rs_check_toolchain
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_signed_integer # Run benchmarks for signed integer
 bench_signed_integer: install_rs_check_toolchain
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer-signed \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend
 bench_integer_gpu: install_rs_check_toolchain
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
 
 .PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend
 bench_signed_integer_gpu: install_rs_check_toolchain
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer-signed \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
@@ -1383,7 +1383,7 @@ bench_signed_integer_gpu: install_rs_check_toolchain
 bench_integer_hpu: install_rs_check_toolchain
 	source ./setup_hpu.sh --config $(HPU_CONFIG); \
 	export V80_PCIE_DEV=${V80_PCIE_DEV}; \
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer \
 	--features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick
@@ -1433,7 +1433,7 @@ bench_integer_aes256_gpu: install_rs_check_toolchain
 .PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters
 bench_integer_multi_bit: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
-	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
+	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
@@ -1441,7 +1441,7 @@ bench_integer_multi_bit: install_rs_check_toolchain
 .PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters
 bench_signed_integer_multi_bit: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
-	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
+	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer-signed \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
@@ -1449,7 +1449,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain
 .PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters
 bench_integer_multi_bit_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
-	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
@@ -1457,7 +1457,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain
 .PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters
 bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
-	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench integer-signed \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
@@ -1513,7 +1513,7 @@ bench_pbs: install_rs_check_toolchain
 
 .PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend
 bench_pbs_gpu: install_rs_check_toolchain
-	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
+	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
 	--bench core_crypto-pbs \
 	--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark --profile release_lto_off

tfhe-benchmark/src/utilities.rs

@@ -351,12 +351,33 @@ const BENCH_BIT_SIZES: [usize; 8] = [2, 8, 16, 32, 40, 64, 128, 256];
 const BENCH_BIT_SIZES: [usize; 7] = [8, 16, 32, 40, 64, 128, 256];
 const HPU_BENCH_BIT_SIZES: [usize; 5] = [8, 16, 32, 64, 128];
 const MULTI_BIT_CPU_SIZES: [usize; 5] = [8, 16, 32, 40, 64];
+const BENCH_BIT_SIZES_DOCUMENTATION: [usize; 5] = [8, 16, 32, 64, 128];
+
+#[derive(Default)]
+pub enum BitSizesSet {
+    #[default]
+    Fast,
+    All,
+    Documentation,
+}
+
+impl BitSizesSet {
+    pub fn from_env() -> Result<Self, String> {
+        let raw_value = env::var("__TFHE_RS_BENCH_BIT_SIZES_SET").unwrap_or("fast".to_string());
+        match raw_value.to_lowercase().as_str() {
+            "fast" => Ok(BitSizesSet::Fast),
+            "all" => Ok(BitSizesSet::All),
+            "documentation" => Ok(BitSizesSet::Documentation),
+            _ => Err(format!("bit sizes set '{raw_value}' is not supported")),
+        }
+    }
+}
 
 /// User configuration in which benchmarks must be run.
 #[derive(Default)]
 pub struct EnvConfig {
     pub is_multi_bit: bool,
-    pub is_fast_bench: bool,
+    pub bit_sizes_set: BitSizesSet,
 }
 
 impl EnvConfig {
@@ -366,22 +387,21 @@ impl EnvConfig {
             ParamType::MultiBit | ParamType::MultiBitDocumentation
         );
 
-        let is_fast_bench = match env::var("__TFHE_RS_FAST_BENCH") {
-            Ok(val) => val.to_lowercase() == "true",
-            Err(_) => false,
-        };
-
         EnvConfig {
             is_multi_bit,
-            is_fast_bench,
+            bit_sizes_set: BitSizesSet::from_env().unwrap(),
         }
     }
 
     /// Get precisions values to benchmark.
     pub fn bit_sizes(&self) -> Vec<usize> {
-        if self.is_fast_bench {
+        let bit_sizes_set = match self.bit_sizes_set {
-            FAST_BENCH_BIT_SIZES.to_vec()
+            BitSizesSet::Fast => return FAST_BENCH_BIT_SIZES.to_vec(),
-        } else if self.is_multi_bit {
+            BitSizesSet::All => BENCH_BIT_SIZES.to_vec(),
+            BitSizesSet::Documentation => return BENCH_BIT_SIZES_DOCUMENTATION.to_vec(),
+        };
+
+        if self.is_multi_bit {
             if cfg!(feature = "gpu") {
                 BENCH_BIT_SIZES.to_vec()
             } else {
@@ -390,7 +410,7 @@ impl EnvConfig {
         } else if cfg!(feature = "hpu") {
             HPU_BENCH_BIT_SIZES.to_vec()
         } else {
-            BENCH_BIT_SIZES.to_vec()
+            bit_sizes_set
         }
     }
 }