diff --git a/.github/workflows/benchmark_hpu.yml b/.github/workflows/benchmark_hpu.yml new file mode 100644 index 000000000..a95aff409 --- /dev/null +++ b/.github/workflows/benchmark_hpu.yml @@ -0,0 +1,63 @@ +# Run benchmarks on a permanent HPU instance and return parsed results to Slab CI bot. +name: benchmark_hpu + +on: + workflow_dispatch: + inputs: + command: + description: "Benchmark command to run" + type: choice + default: integer + options: + - integer + - hlapi + - hlapi_erc20 + op_flavor: + description: "Operations set to run" + type: choice + default: default + options: + - default + - fast_default + all_precisions: + description: "Run all precisions" + type: boolean + default: false + bench_type: + description: "Benchmarks type" + type: choice + default: latency + options: + - latency + - throughput + - both + v80_pcie_dev: + description: "V80 PCIe device number" + default: 24 + v80_serial_number: + description: "V80 serial number" + default: XFL12NWY3ZKG + +permissions: {} + +jobs: + run-benchmarks: + name: benchmark_hpu/run-benchmarks + uses: ./.github/workflows/benchmark_hpu_common.yml + with: + command: ${{ inputs.command }} + op_flavor: ${{ inputs.op_flavor }} + bench_type: ${{ inputs.bench_type }} + all_precisions: ${{ inputs.all_precisions }} + v80_pcie_dev: ${{ inputs.v80_pcie_dev }} + v80_serial_number: ${{ inputs.v80_serial_number }} + secrets: + BOT_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} + JOB_SECRET: ${{ secrets.JOB_SECRET }} + SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }} + SLAB_URL: ${{ secrets.SLAB_URL }} + SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} diff --git a/.github/workflows/benchmark_hpu_integer.yml b/.github/workflows/benchmark_hpu_common.yml similarity index 53% rename from .github/workflows/benchmark_hpu_integer.yml rename to .github/workflows/benchmark_hpu_common.yml index b049a7aac..2b5009f67 100644 --- a/.github/workflows/benchmark_hpu_integer.yml +++ b/.github/workflows/benchmark_hpu_common.yml @@ -1,20 +1,46 @@ -# Run all integer benchmarks on a permanent HPU instance and return parsed results to Slab CI bot. -name: benchmark_hpu_integer +# Run benchmarks on a permanent HPU instance and return parsed results to Slab CI bot. +name: benchmark_hpu_common on: - workflow_dispatch: + workflow_call: inputs: - all_precisions: - description: "Run all precisions" - type: boolean + command: # Use a comma separated values to generate an array + type: string + required: true + op_flavor: # Use a comma separated values to generate an array + type: string + default: default bench_type: - description: "Benchmarks type" - type: choice - default: both - options: - - latency - - throughput - - both + type: string + default: latency + all_precisions: + type: boolean + default: false + v80_pcie_dev: + type: string + default: 24 + v80_serial_number: + type: string + default: XFL12NWY3ZKG + secrets: + REPO_CHECKOUT_TOKEN: + required: true + SLAB_ACTION_TOKEN: + required: true + SLAB_BASE_URL: + required: true + SLAB_URL: + required: true + JOB_SECRET: + required: true + SLACK_CHANNEL: + required: true + BOT_USERNAME: + required: true + SLACK_WEBHOOK: + required: true + SSH_PRIVATE_KEY: + required: true env: CARGO_TERM_COLOR: always @@ -28,35 +54,46 @@ permissions: {} jobs: prepare-matrix: - name: Prepare operations matrix - runs-on: v80-marais + name: benchmark_hpu_common/prepare-matrix + runs-on: ubuntu-latest outputs: - bench_type: ${{ steps.set_bench_type.outputs.bench_type }} + command: ${{ steps.set_matrix_args.outputs.command }} + op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }} + bench_type: ${{ steps.set_matrix_args.outputs.bench_type }} + env: + INPUTS_COMMAND: ${{ inputs.command }} + INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }} steps: - - name: Set benchmark types - if: github.event_name == 'workflow_dispatch' - run: | - if [[ -z $INPUTS_BENCH_TYPE || "${INPUTS_BENCH_TYPE}" == "both" ]]; then - echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}" - else - echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}" - fi - env: - INPUTS_BENCH_TYPE: ${{ inputs.bench_type }} + - name: Parse user inputs + shell: python + run: | # zizmor: ignore[template-injection] these env variables are safe + split_command = "${{ inputs.command }}".replace(" ", "").split(",") + split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",") - - name: Default benchmark type - if: github.event_name != 'workflow_dispatch' - run: | - echo "BENCH_TYPE=[\"latency\"]" >> "${GITHUB_ENV}" - - - name: Set benchmark types output - id: set_bench_type - run: | # zizmor: ignore[template-injection] this env variable is safe - echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}" + if "${{ inputs.bench_type }}" == "both": + bench_type = ["latency", "throughput"] + else: + bench_type = ["${{ inputs.bench_type }}", ] + with open("${{ github.env }}", "a") as f: + for env_name, values_to_join in [ + ("COMMAND", split_command), + ("OP_FLAVOR", split_op_flavor), + ("BENCH_TYPE", bench_type), + ]: + f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""") - integer-benchmarks-hpu: - name: benchmark_hpu_integer/integer-benchmarks-hpu + - name: Set martix arguments outputs + id: set_matrix_args + run: | # zizmor: ignore[template-injection] these env variable are safe + { + echo "command=${{ toJSON(env.COMMAND) }}"; + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}"; + echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}"; + } >> "${GITHUB_OUTPUT}" + + hpu-benchmarks: + name: benchmark_hpu_common/hpu-benchmarks needs: prepare-matrix runs-on: v80-marais concurrency: @@ -66,6 +103,8 @@ jobs: strategy: max-parallel: 1 matrix: + command: ${{ fromJSON(needs.prepare-matrix.outputs.command) }} + op_flavor: ${{ fromJSON(needs.prepare-matrix.outputs.op_flavor) }} bench_type: ${{ fromJSON(needs.prepare-matrix.outputs.bench_type) }} steps: # Needed as long as hw_regmap repository is private @@ -98,14 +137,6 @@ jobs: with: toolchain: nightly - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - name: Should run benchmarks with all precisions if: inputs.all_precisions run: | @@ -113,16 +144,21 @@ jobs: - name: Select HPU board run: | - echo "V80_PCIE_DEV=24" >> "${GITHUB_ENV}" - echo "V80_SERIAL_NUMBER=XFL12NWY3ZKG" >> "${GITHUB_ENV}" + echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}" + echo "V80_SERIAL_NUMBER=${SERIAL_NUMBER}" >> "${GITHUB_ENV}" + env: + PCIE_DEV: ${{ inputs.v80_pcie_dev }} + SERIAL_NUMBER: ${{ inputs.v80_serial_number }} - name: Run benchmarks run: | echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}" make pull_hpu_files - make BENCH_TYPE="${BENCH_TYPE}" bench_integer_hpu + make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu env: + OP_FLAVOR: ${{ matrix.op_flavor }} BENCH_TYPE: ${{ matrix.bench_type }} + BENCH_COMMAND: ${{ matrix.command }} - name: Parse results run: | @@ -146,6 +182,14 @@ jobs: name: ${{ github.sha }}_${{ matrix.bench_type }}_integer_benchmarks path: ${{ env.RESULTS_FILENAME }} + - name: Checkout Slab repo + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + with: + repository: zama-ai/slab + path: slab + persist-credentials: 'false' + token: ${{ secrets.REPO_CHECKOUT_TOKEN }} + - name: Send data to Slab shell: bash run: | diff --git a/.github/workflows/benchmark_hpu_hlapi.yml b/.github/workflows/benchmark_hpu_hlapi.yml deleted file mode 100644 index ecaaa14f9..000000000 --- a/.github/workflows/benchmark_hpu_hlapi.yml +++ /dev/null @@ -1,101 +0,0 @@ -# Run all integer benchmarks on a permanent HPU instance and return parsed results to Slab CI bot. -name: Hpu Hlapi Benchmarks - -on: - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -permissions: {} - -jobs: - hlapi-benchmarks-hpu: - name: Execute HLAPI benchmarks for HPU backend - runs-on: v80-marais - concurrency: - group: ${{ github.workflow }}_${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - timeout-minutes: 1440 # 24 hours - steps: - # Needed as long as hw_regmap repository is private - - name: Configure SSH - uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - fetch-depth: 0 - persist-credentials: 'false' - lfs: true - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Get benchmark details - run: | - COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=${COMMIT_DATE}"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - env: - SHA: ${{ github.sha }} - - - name: Install rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[stale-action-refs] this action doesn't create releases - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 - with: - repository: zama-ai/slab - path: slab - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - - name: Select HPU board - run: | - echo "V80_PCIE_DEV=24" >> "${GITHUB_ENV}" - echo "V80_SERIAL_NUMBER=XFL12NWY3ZKG" >> "${GITHUB_ENV}" - - - name: Run benchmarks - run: | - make pull_hpu_files - make bench_hlapi_erc20_hpu - make bench_hlapi_hpu - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ - --database tfhe_rs \ - --hardware "hpu_x1" \ - --backend hpu \ - --project-version "${COMMIT_HASH}" \ - --branch "${REF_NAME}" \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${BENCH_DATE}" \ - --walk-subdirs - env: - REF_NAME: ${{ github.ref_name }} - - - name: Upload parsed results artifact - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 - with: - name: ${{ github.sha }}_hlapi_benchmarks - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ - --slab-url "${SLAB_URL}" - env: - JOB_SECRET: ${{ secrets.JOB_SECRET }} - SLAB_URL: ${{ secrets.SLAB_URL }}