tfhe-rs/.github/workflows/benchmark_documentation.yml

# Run all benchmarks displayed in the public documentation.
name: benchmark_documentation

on:
  workflow_dispatch:
    inputs:
      run-cpu-benchmarks:
        description: "Run CPU benchmarks"
        type: boolean
        default: true
      run-gpu-benchmarks:
        description: "Run GPU benchmarks"
        type: boolean
        default: true
      run-hpu-benchmarks:
        description: "Run HPU benchmarks"
        type: boolean
        default: true
      generate-svgs:
        description: "Generate SVG tables"
        type: boolean
        default: true
      open-pr:
        description: "Open a PR with the benchmark results"
        type: boolean
        default: false

permissions: {}

# zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow

jobs:
  run-benchmarks-cpu-integer:
    name: benchmark_documentation/run-benchmarks-cpu-integer
    uses: ./.github/workflows/benchmark_cpu_common.yml
    if: inputs.run-cpu-benchmarks
    with:
      command: integer
      op_flavor: fast_default
      bench_type: both
      precisions_set: documentation
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}

  run-benchmarks-gpu-integer:
    name: benchmark_documentation/run-benchmarks-gpu-integer
    uses: ./.github/workflows/benchmark_gpu_common.yml
    if: inputs.run-gpu-benchmarks
    with:
      profile: multi-h100-sxm5
      hardware_name: n3-H100-SXM5x8
      command: integer_multi_bit
      op_flavor: fast_default
      bench_type: both
      precisions_set: documentation
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}

  run-benchmarks-hpu-integer:
    name: benchmark_documentation/run-benchmarks-hpu-integer
    uses: ./.github/workflows/benchmark_hpu_common.yml
    if: inputs.run-hpu-benchmarks
    with:
      command: integer
      op_flavor: default
      bench_type: both
      precisions_set: documentation
      v80_pcie_dev: 24
      v80_serial_number: XFL12NWY3ZKG
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
      SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

  run-benchmarks-cpu-core-crypto:
    name: benchmark_documentation/run-benchmarks-cpu-core-crypto
    uses: ./.github/workflows/benchmark_cpu_common.yml
    if: inputs.run-cpu-benchmarks
    with:
      command: pbs, ks_pbs
      bench_type: latency
      params_type: classical_documentation + multi_bit_documentation
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}

  run-benchmarks-gpu-core-crypto:
    name: benchmark_documentation/run-benchmarks-gpu-core-crypto
    uses: ./.github/workflows/benchmark_gpu_common.yml
    if: inputs.run-gpu-benchmarks
    with:
      profile: multi-h100-sxm5
      hardware_name: n3-H100-SXM5x8
      command: pbs, ks_pbs
      bench_type: latency
      params_type: classical_documentation + multi_bit_documentation
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}

  generate-svgs-with-benchmarks-run:
    name: benchmark-documentation/generate-svgs-with-benchmarks-run
    if: ${{ always() &&
      (inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) &&
      inputs.generate-svgs }}
    needs: [
      run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
      run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
    ]
    uses: ./.github/workflows/generate_svgs.yml
    with:
      time_span_days: 5
      generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }}
      generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }}
      generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }}
    secrets:
      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}

  generate-svgs-without-benchmarks-run:
    name: benchmark-documentation/generate-svgs-without-benchmarks-run
    if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) &&
      inputs.generate-svgs }}
    uses: ./.github/workflows/generate_svgs.yml
    with:
      time_span_days: 60
    secrets:
      DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
      DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
      DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}

  open-pr:
    name: benchmark-documentation/open-pr
    needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ]
    if: ${{ always() && inputs.open-pr &&
      (needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }}
    runs-on: ubuntu-latest
    permissions:
      contents: write # Needed to create a commit
      pull-requests: write # Needed to open a pull-request
    env:
      PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets
    steps:
      - name: Checkout tfhe-rs
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
        with:
          persist-credentials: 'false'

      - name: Download SVG tables
        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
        with:
          path: svg_tables
          merge-multiple: 'true'

      # Perform best effort to copy SVG tables. If the copy fails or files don't exist, the PR will still be created.
      - name: Copy SVG tables to documentation location
        run: |
          cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}" 2>/dev/null
          cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}" 2>/dev/null
          cp -f svg_tables/cpu-gpu-hpu-integer-benchmark-fheuint64-tuniform-2m128-ciphertext.svg "${PATH_TO_DOC_ASSETS}" 2>/dev/null

      - name: Get current date
        id: get-date
        run: |
          echo "date=$(date '+%g_%m_%d_%Hh%Mm%Ss')" >> "${GITHUB_OUTPUT}"

      - name: Create pull-request
        uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9
        with:
          sign-commits: true # Commit will be signed by github-actions bot
          add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg
          branch: gh-bot/docs/update-svg-tables-${{ steps.get-date.outputs.date }}
          commit-message: |
            chore(docs): update benchmark results for all backends

            Automated documentation update from tfhe-rs CI pipeline.
          title: |
            [CI] chore(docs): update benchmark results for all backends
          body: |
            Documentation update triggered by GitHub workflow.
          labels: documentation