mirror of
https://github.com/zama-ai/tfhe-rs.git
synced 2026-01-06 21:34:05 -05:00
chore(ci): add workflow to update documentation benchmark tables
This new workflow can trigger all the required benchmarks needed to populate benchmarks tables in documentation. It also can generate SVG tables and store them as artifacts. Optionally, it can open a pull-request to update the current tables in documentation.
This commit is contained in:
David Testé
14
.github/workflows/benchmark_cpu.yml
vendored
14
.github/workflows/benchmark_cpu.yml
vendored
@@ -34,10 +34,14 @@ on:
|
||||
- smart
|
||||
- unchecked
|
||||
- misc
|
||||
all_precisions:
|
||||
description: "Run all precisions"
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
description: "Bit precisions set"
|
||||
type: choice
|
||||
default: fast
|
||||
options:
|
||||
- fast
|
||||
- all
|
||||
- documentation
|
||||
bench_type:
|
||||
description: "Benchmarks type"
|
||||
type: choice
|
||||
@@ -71,7 +75,7 @@ jobs:
|
||||
op_flavor: ${{ inputs.op_flavor }}
|
||||
bench_type: ${{ inputs.bench_type }}
|
||||
params_type: ${{ inputs.params_type }}
|
||||
all_precisions: ${{ inputs.all_precisions }}
|
||||
precisions_set: ${{ inputs.precisions_set }}
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
|
||||
18
.github/workflows/benchmark_cpu_common.yml
vendored
18
.github/workflows/benchmark_cpu_common.yml
vendored
@@ -16,9 +16,9 @@ on:
|
||||
params_type:
|
||||
type: string
|
||||
default: classical
|
||||
all_precisions:
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
type: string
|
||||
default: fast
|
||||
additional_recipe: # Make recipes to run aside the benchmarks.
|
||||
type: string # Use comma separated values to generate an array
|
||||
additional_file_to_parse: # Other files to parse, located under tfhe-benchmark/ directory
|
||||
@@ -54,7 +54,6 @@ env:
|
||||
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
||||
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
FAST_BENCH: TRUE
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -69,9 +68,6 @@ jobs:
|
||||
op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
|
||||
bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
|
||||
params_type: ${{ steps.set_matrix_args.outputs.params_type }}
|
||||
env:
|
||||
INPUTS_COMMAND: ${{ inputs.command }}
|
||||
INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
|
||||
steps:
|
||||
- name: Parse user inputs
|
||||
shell: python
|
||||
@@ -162,19 +158,15 @@ jobs:
|
||||
with:
|
||||
toolchain: nightly
|
||||
|
||||
- name: Should run benchmarks with all precisions
|
||||
if: inputs.all_precisions
|
||||
run: |
|
||||
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Run benchmarks with AVX512
|
||||
run: |
|
||||
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
|
||||
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"
|
||||
env:
|
||||
OP_FLAVOR: ${{ matrix.op_flavor }}
|
||||
BENCH_TYPE: ${{ matrix.bench_type }}
|
||||
BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
|
||||
BENCH_COMMAND: ${{ matrix.command }}
|
||||
PRECISIONS_SET: ${{ inputs.precisions_set }}
|
||||
|
||||
- name: Parse results
|
||||
run: |
|
||||
|
||||
12
.github/workflows/benchmark_cpu_weekly.yml
vendored
12
.github/workflows/benchmark_cpu_weekly.yml
vendored
@@ -30,7 +30,7 @@ jobs:
|
||||
is_weekly_bench_group_2: ${{ steps.check_bench_group_2.outputs.is_weekly_bench_group_2 }}
|
||||
is_quarterly_bench: ${{ steps.check_quarterly_bench.outputs.is_quarterly_bench }}
|
||||
op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
|
||||
all_precisions: ${{ steps.set_all_precisions.outputs.all_precisions }}
|
||||
precisions_set: ${{ steps.set_precisions_set.outputs.precisions_set }}
|
||||
steps:
|
||||
- name: Check is weekly bench group 1
|
||||
id: check_bench_group_1
|
||||
@@ -52,13 +52,13 @@ jobs:
|
||||
steps.check_bench_group_2.outputs.is_weekly_bench_group_2 == 'true'
|
||||
run: |
|
||||
echo "OP_FLAVOR=default" >> "${GITHUB_ENV}"
|
||||
echo "ALL_PRECISIONS=false" >> "${GITHUB_ENV}"
|
||||
echo "PRECISIONS_SET=false" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Quarterly benchmarks
|
||||
if: steps.check_quarterly_bench.outputs.is_quarterly_bench == 'true'
|
||||
run: |
|
||||
echo "OP_FLAVOR=\"default,unchecked\"" >> "${GITHUB_ENV}"
|
||||
echo "ALL_PRECISIONS=true" >> "${GITHUB_ENV}"
|
||||
echo "PRECISIONS_SET=true" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set operation flavor output
|
||||
id: set_op_flavor
|
||||
@@ -66,9 +66,9 @@ jobs:
|
||||
echo "op_flavor=${{ env.OP_FLAVOR }}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Set bit precisions output
|
||||
id: set_all_precisions
|
||||
id: set_precisions_set
|
||||
run: | # zizmor: ignore[template-injection] this env variable is safe
|
||||
echo "all_precisions=${{ env.ALL_PRECISIONS }}" >> "${GITHUB_OUTPUT}"
|
||||
echo "precisions_set=${{ env.PRECISIONS_SET }}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
run-benchmarks-integer:
|
||||
name: benchmark_cpu_weekly/run-benchmarks-integer
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
with:
|
||||
command: integer,signed_integer, integer_compression
|
||||
op_flavor: ${{ needs.prepare-inputs.outputs.op_flavor }}
|
||||
all_precisions: ${{ needs.prepare-inputs.outputs.all_precisions }}
|
||||
precisions_set: ${{ needs.prepare-inputs.outputs.precisions_set }}
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
|
||||
207
.github/workflows/benchmark_documentation.yml
vendored
207
.github/workflows/benchmark_documentation.yml
vendored
@@ -1,17 +1,212 @@
|
||||
name: benchmark_documentation.yml
|
||||
# Run all benchmarks displayed in the public documentation.
|
||||
name: benchmark_documentation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
run-cpu-benchmarks:
|
||||
description: "Run CPU benchmarks"
|
||||
type: boolean
|
||||
default: true
|
||||
run-gpu-benchmarks:
|
||||
description: "Run GPU benchmarks"
|
||||
type: boolean
|
||||
default: true
|
||||
run-hpu-benchmarks:
|
||||
description: "Run HPU benchmarks"
|
||||
type: boolean
|
||||
default: true
|
||||
generate-svgs:
|
||||
description: "Generate SVG tables"
|
||||
type: boolean
|
||||
default: true
|
||||
open-pr:
|
||||
description: "Open a PR with the benchmark results"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
# zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow
|
||||
|
||||
jobs:
|
||||
placeholder:
|
||||
name: benchmark_documentation/placeholder
|
||||
runs-on: ubuntu-latest
|
||||
run-benchmarks-cpu-integer:
|
||||
name: benchmark_documentation/run-benchmarks-cpu-integer
|
||||
uses: ./.github/workflows/benchmark_cpu_common.yml
|
||||
if: inputs.run-cpu-benchmarks
|
||||
with:
|
||||
command: integer
|
||||
op_flavor: fast_default
|
||||
bench_type: both
|
||||
precisions_set: documentation
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
|
||||
|
||||
run-benchmarks-gpu-integer:
|
||||
name: benchmark_documentation/run-benchmarks-gpu-integer
|
||||
uses: ./.github/workflows/benchmark_gpu_common.yml
|
||||
if: inputs.run-gpu-benchmarks
|
||||
with:
|
||||
profile: multi-h100-sxm5
|
||||
hardware_name: n3-H100-SXM5x8
|
||||
command: integer_multi_bit
|
||||
op_flavor: fast_default
|
||||
bench_type: both
|
||||
precisions_set: documentation
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
|
||||
|
||||
run-benchmarks-hpu-integer:
|
||||
name: benchmark_documentation/run-benchmarks-hpu-integer
|
||||
uses: ./.github/workflows/benchmark_hpu_common.yml
|
||||
if: inputs.run-hpu-benchmarks
|
||||
with:
|
||||
command: integer
|
||||
op_flavor: default
|
||||
bench_type: both
|
||||
precisions_set: documentation
|
||||
v80_pcie_dev: 24
|
||||
v80_serial_number: XFL12NWY3ZKG
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
|
||||
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
|
||||
run-benchmarks-cpu-core-crypto:
|
||||
name: benchmark_documentation/run-benchmarks-cpu-core-crypto
|
||||
uses: ./.github/workflows/benchmark_cpu_common.yml
|
||||
if: inputs.run-cpu-benchmarks
|
||||
with:
|
||||
command: pbs, ks_pbs
|
||||
bench_type: latency
|
||||
params_type: classical_documentation + multi_bit_documentation
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
|
||||
|
||||
run-benchmarks-gpu-core-crypto:
|
||||
name: benchmark_documentation/run-benchmarks-gpu-core-crypto
|
||||
uses: ./.github/workflows/benchmark_gpu_common.yml
|
||||
if: inputs.run-gpu-benchmarks
|
||||
with:
|
||||
profile: multi-h100-sxm5
|
||||
hardware_name: n3-H100-SXM5x8
|
||||
command: pbs, ks_pbs
|
||||
bench_type: latency
|
||||
params_type: classical_documentation + multi_bit_documentation
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}
|
||||
|
||||
generate-svgs-with-benchmarks-run:
|
||||
name: benchmark-documentation/generate-svgs-with-benchmarks-run
|
||||
if: ${{ always() &&
|
||||
(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks ||inputs.run-hpu-benchmarks) &&
|
||||
inputs.generate-svgs }}
|
||||
needs: [
|
||||
run-benchmarks-cpu-integer, run-benchmarks-gpu-integer, run-benchmarks-hpu-integer,
|
||||
run-benchmarks-cpu-core-crypto, run-benchmarks-gpu-core-crypto
|
||||
]
|
||||
uses: ./.github/workflows/generate_svgs.yml
|
||||
with:
|
||||
time_span_days: 5
|
||||
generate-cpu-svgs: ${{ inputs.run-cpu-benchmarks }}
|
||||
generate-gpu-svgs: ${{ inputs.run-gpu-benchmarks }}
|
||||
generate-hpu-svgs: ${{ inputs.run-hpu-benchmarks }}
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
generate-svgs-without-benchmarks-run:
|
||||
name: benchmark-documentation/generate-svgs-without-benchmarks-run
|
||||
if: ${{ !(inputs.run-cpu-benchmarks || inputs.run-gpu-benchmarks || inputs.run-hpu-benchmarks) &&
|
||||
inputs.generate-svgs }}
|
||||
uses: ./.github/workflows/generate_svgs.yml
|
||||
with:
|
||||
time_span_days: 60
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
open-pr:
|
||||
name: benchmark-documentation/open-pr
|
||||
needs: [ generate-svgs-with-benchmarks-run, generate-svgs-without-benchmarks-run ]
|
||||
if: ${{ always() && inputs.open-pr &&
|
||||
(needs.generate-svgs-with-benchmarks-run.result == 'success' || needs.generate-svgs-without-benchmarks-run.result == 'success') }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # Needed to create a commit
|
||||
pull-requests: write # Needed to open a pull-request
|
||||
env:
|
||||
PATH_TO_DOC_ASSETS: tfhe/docs/.gitbook/assets
|
||||
steps:
|
||||
- run: |
|
||||
echo "Hello this is a placeholder workflow"
|
||||
- name: Checkout tfhe-rs
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
with:
|
||||
persist-credentials: 'false'
|
||||
|
||||
- name: Download SVG tables
|
||||
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
|
||||
with:
|
||||
path: svg_tables
|
||||
merge-multiple: 'true'
|
||||
|
||||
- name: Copy SVG tables to documentation location
|
||||
run: |
|
||||
cp -f svg_tables/*integer-benchmark*.svg "${PATH_TO_DOC_ASSETS}"
|
||||
cp -f svg_tables/*pbs-benchmark-tuniform*.svg "${PATH_TO_DOC_ASSETS}"
|
||||
|
||||
- name: Get current date
|
||||
id: get-date
|
||||
run: |
|
||||
echo "date=$(date '+%g_%m_%d_%Hh%Mm%Ss')" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Create pull-request
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
sign-commits: true # Commit will be signed by github-actions bot
|
||||
add-paths: ${{ env.PATH_TO_DOC_ASSETS }}/*.svg
|
||||
branch: gh-bot/docs/update-svg-tables-${{ steps.get-date.outputs.date }}
|
||||
commit-message: |
|
||||
chore(docs): update benchmark results for all backends
|
||||
|
||||
Automated documentation update from tfhe-rs CI pipeline.
|
||||
title: |
|
||||
[CI] chore(docs): update benchmark results for all backends
|
||||
body: |
|
||||
Documentation update triggered by GitHub workflow.
|
||||
labels: documentation
|
||||
|
||||
14
.github/workflows/benchmark_gpu.yml
vendored
14
.github/workflows/benchmark_gpu.yml
vendored
@@ -42,10 +42,14 @@ on:
|
||||
- default
|
||||
- fast_default
|
||||
- unchecked
|
||||
all_precisions:
|
||||
description: "Run all precisions"
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
description: "Bit precisions set"
|
||||
type: choice
|
||||
default: fast
|
||||
options:
|
||||
- fast
|
||||
- all
|
||||
- documentation
|
||||
bench_type:
|
||||
description: "Benchmarks type"
|
||||
type: choice
|
||||
@@ -108,7 +112,7 @@ jobs:
|
||||
op_flavor: ${{ inputs.op_flavor }}
|
||||
bench_type: ${{ inputs.bench_type }}
|
||||
params_type: ${{ inputs.params_type }}
|
||||
all_precisions: ${{ inputs.all_precisions }}
|
||||
precisions_set: ${{ inputs.precisions_set }}
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
|
||||
3
.github/workflows/benchmark_gpu_4090.yml
vendored
3
.github/workflows/benchmark_gpu_4090.yml
vendored
@@ -11,7 +11,7 @@ env:
|
||||
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
||||
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
FAST_BENCH: TRUE
|
||||
BIT_SIZES_SET: FAST
|
||||
|
||||
on:
|
||||
# Allows you to run this workflow manually from the Actions tab as an alternative.
|
||||
@@ -53,7 +53,6 @@ jobs:
|
||||
echo "BENCH_DATE=$(date --iso-8601=seconds)";
|
||||
echo "COMMIT_DATE=${COMMIT_DATE}";
|
||||
echo "COMMIT_HASH=$(git describe --tags --dirty)";
|
||||
echo "FAST_BENCH=TRUE";
|
||||
} >> "${GITHUB_ENV}"
|
||||
env:
|
||||
SHA: ${{ github.sha }}
|
||||
|
||||
118
.github/workflows/benchmark_gpu_common.yml
vendored
118
.github/workflows/benchmark_gpu_common.yml
vendored
@@ -25,9 +25,9 @@ on:
|
||||
params_type:
|
||||
type: string
|
||||
default: multi_bit
|
||||
all_precisions:
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
type: string
|
||||
default: fast
|
||||
secrets:
|
||||
REPO_CHECKOUT_TOKEN:
|
||||
required: true
|
||||
@@ -56,8 +56,6 @@ env:
|
||||
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
||||
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
||||
FAST_BENCH: TRUE
|
||||
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -68,79 +66,45 @@ jobs:
|
||||
name: benchmark_gpu_common/prepare-matrix
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
command: ${{ steps.set_command.outputs.command }}
|
||||
op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }}
|
||||
bench_type: ${{ steps.set_bench_type.outputs.bench_type }}
|
||||
params_type: ${{ steps.set_params_type.outputs.params_type }}
|
||||
env:
|
||||
INPUTS_COMMAND: ${{ inputs.command }}
|
||||
INPUTS_OP_FLAVOR: ${{ inputs.op_flavor }}
|
||||
command: ${{ steps.set_matrix_args.outputs.command }}
|
||||
op_flavor: ${{ steps.set_matrix_args.outputs.op_flavor }}
|
||||
bench_type: ${{ steps.set_matrix_args.outputs.bench_type }}
|
||||
params_type: ${{ steps.set_matrix_args.outputs.params_type }}
|
||||
steps:
|
||||
- name: Set single command
|
||||
if: ${{ !contains(inputs.command, ',')}}
|
||||
run: |
|
||||
echo "COMMAND=[\"${INPUTS_COMMAND}\"]" >> "${GITHUB_ENV}"
|
||||
- name: Parse user inputs
|
||||
shell: python
|
||||
run: | # zizmor: ignore[template-injection] these env variables are safe
|
||||
split_command = "${{ inputs.command }}".replace(" ", "").split(",")
|
||||
split_op_flavor = "${{ inputs.op_flavor }}".replace(" ", "").split(",")
|
||||
|
||||
- name: Set multiple commands
|
||||
if: ${{ contains(inputs.command, ',')}}
|
||||
run: |
|
||||
# Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
|
||||
# shellcheck disable=SC2001
|
||||
PARSED_COMMAND=$(echo "${INPUTS_COMMAND}" | sed 's/[[:space:]]*,[[:space:]]*/\", \"/g')
|
||||
echo "COMMAND=[\"${PARSED_COMMAND}\"]" >> "${GITHUB_ENV}"
|
||||
if "${{ inputs.bench_type }}" == "both":
|
||||
bench_type = ["latency", "throughput"]
|
||||
else:
|
||||
bench_type = ["${{ inputs.bench_type }}", ]
|
||||
|
||||
- name: Set single operations flavor
|
||||
if: ${{ !contains(inputs.op_flavor, ',')}}
|
||||
run: |
|
||||
echo "OP_FLAVOR=[\"${INPUTS_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
|
||||
if "+" in "${{ inputs.params_type }}":
|
||||
split_params_type= "${{ inputs.params_type }}".replace(" ", "").split("+")
|
||||
else:
|
||||
split_params_type = ["${{ inputs.params_type }}", ]
|
||||
|
||||
- name: Set multiple operations flavors
|
||||
if: ${{ contains(inputs.op_flavor, ',')}}
|
||||
run: |
|
||||
# Use Sed to extract a value from a string, this cannot be done with the ${variable//search/replace} pattern.
|
||||
# shellcheck disable=SC2001
|
||||
PARSED_OP_FLAVOR=$(echo "${INPUTS_OP_FLAVOR}" | sed 's/[[:space:]]*,[[:space:]]*/", "/g')
|
||||
echo "OP_FLAVOR=[\"${PARSED_OP_FLAVOR}\"]" >> "${GITHUB_ENV}"
|
||||
with open("${{ github.env }}", "a") as f:
|
||||
for env_name, values_to_join in [
|
||||
("COMMAND", split_command),
|
||||
("OP_FLAVOR", split_op_flavor),
|
||||
("BENCH_TYPE", bench_type),
|
||||
("PARAMS_TYPE", split_params_type),
|
||||
]:
|
||||
f.write(f"""{env_name}=["{'", "'.join(values_to_join)}"]\n""")
|
||||
|
||||
- name: Set benchmark types
|
||||
run: |
|
||||
if [[ "${INPUTS_BENCH_TYPE}" == "both" ]]; then
|
||||
echo "BENCH_TYPE=[\"latency\", \"throughput\"]" >> "${GITHUB_ENV}"
|
||||
else
|
||||
echo "BENCH_TYPE=[\"${INPUTS_BENCH_TYPE}\"]" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
env:
|
||||
INPUTS_BENCH_TYPE: ${{ inputs.bench_type }}
|
||||
|
||||
- name: Set parameters types
|
||||
run: |
|
||||
if [[ "${INPUTS_PARAMS_TYPE}" == "both" ]]; then
|
||||
echo "PARAMS_TYPE=[\"classical\", \"multi_bit\"]" >> "${GITHUB_ENV}"
|
||||
else
|
||||
echo "PARAMS_TYPE=[\"${INPUTS_PARAMS_TYPE}\"]" >> "${GITHUB_ENV}"
|
||||
fi
|
||||
env:
|
||||
INPUTS_PARAMS_TYPE: ${{ inputs.params_type }}
|
||||
|
||||
- name: Set command output
|
||||
id: set_command
|
||||
run: | # zizmor: ignore[template-injection] this env variable is safe
|
||||
echo "command=${{ toJSON(env.COMMAND) }}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Set operation flavor output
|
||||
id: set_op_flavor
|
||||
run: | # zizmor: ignore[template-injection] this env variable is safe
|
||||
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Set benchmark types output
|
||||
id: set_bench_type
|
||||
run: | # zizmor: ignore[template-injection] this env variable is safe
|
||||
echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Set parameters types output
|
||||
id: set_params_type
|
||||
run: | # zizmor: ignore[template-injection] this env variable is safe
|
||||
echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}" >> "${GITHUB_OUTPUT}"
|
||||
- name: Set martix arguments outputs
|
||||
id: set_matrix_args
|
||||
run: | # zizmor: ignore[template-injection] these env variable are safe
|
||||
{
|
||||
echo "command=${{ toJSON(env.COMMAND) }}";
|
||||
echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}";
|
||||
echo "bench_type=${{ toJSON(env.BENCH_TYPE) }}";
|
||||
echo "params_type=${{ toJSON(env.PARAMS_TYPE) }}";
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
|
||||
setup-instance:
|
||||
name: benchmark_gpu_common/setup-instance
|
||||
@@ -275,19 +239,15 @@ jobs:
|
||||
with:
|
||||
toolchain: nightly
|
||||
|
||||
- name: Should run benchmarks with all precisions
|
||||
if: inputs.all_precisions
|
||||
run: |
|
||||
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Run benchmarks
|
||||
run: |
|
||||
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
|
||||
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_gpu
|
||||
env:
|
||||
OP_FLAVOR: ${{ matrix.op_flavor }}
|
||||
BENCH_TYPE: ${{ matrix.bench_type }}
|
||||
BENCH_PARAMS_TYPE: ${{ matrix.params_type }}
|
||||
BENCH_COMMAND: ${{ matrix.command }}
|
||||
PRECISIONS_SET: ${{ inputs.precisions_set }}
|
||||
|
||||
- name: Parse results
|
||||
run: |
|
||||
|
||||
8
.github/workflows/benchmark_gpu_weekly.yml
vendored
8
.github/workflows/benchmark_gpu_weekly.yml
vendored
@@ -22,7 +22,7 @@ jobs:
|
||||
command: integer_multi_bit
|
||||
op_flavor: default
|
||||
bench_type: both
|
||||
all_precisions: true
|
||||
precisions_set: fast
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
command: integer_compression
|
||||
op_flavor: default
|
||||
bench_type: both
|
||||
all_precisions: true
|
||||
precisions_set: fast
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
command: integer_zk,integer_aes,integer_aes256
|
||||
op_flavor: default
|
||||
bench_type: both
|
||||
all_precisions: true
|
||||
precisions_set: fast
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
command: hlapi_noise_squash
|
||||
op_flavor: default
|
||||
bench_type: both
|
||||
all_precisions: true
|
||||
precisions_set: fast
|
||||
secrets:
|
||||
BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
|
||||
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
||||
|
||||
14
.github/workflows/benchmark_hpu.yml
vendored
14
.github/workflows/benchmark_hpu.yml
vendored
@@ -19,10 +19,14 @@ on:
|
||||
options:
|
||||
- default
|
||||
- fast_default
|
||||
all_precisions:
|
||||
description: "Run all precisions"
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
description: "Bit precisions set"
|
||||
type: choice
|
||||
default: fast
|
||||
options:
|
||||
- fast
|
||||
- all
|
||||
- documentation
|
||||
bench_type:
|
||||
description: "Benchmarks type"
|
||||
type: choice
|
||||
@@ -50,7 +54,7 @@ jobs:
|
||||
command: ${{ inputs.command }}
|
||||
op_flavor: ${{ inputs.op_flavor }}
|
||||
bench_type: ${{ inputs.bench_type }}
|
||||
all_precisions: ${{ inputs.all_precisions }}
|
||||
precisions_set: ${{ inputs.precisions_set }}
|
||||
v80_pcie_dev: ${{ inputs.v80_pcie_dev }}
|
||||
v80_serial_number: ${{ inputs.v80_serial_number }}
|
||||
secrets:
|
||||
|
||||
15
.github/workflows/benchmark_hpu_common.yml
vendored
15
.github/workflows/benchmark_hpu_common.yml
vendored
@@ -13,9 +13,9 @@ on:
|
||||
bench_type:
|
||||
type: string
|
||||
default: latency
|
||||
all_precisions:
|
||||
type: boolean
|
||||
default: false
|
||||
precisions_set:
|
||||
type: string
|
||||
default: fast
|
||||
v80_pcie_dev:
|
||||
type: string
|
||||
default: 24
|
||||
@@ -48,7 +48,6 @@ env:
|
||||
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
RUST_BACKTRACE: "full"
|
||||
RUST_MIN_STACK: "8388608"
|
||||
FAST_BENCH: TRUE
|
||||
|
||||
permissions: {}
|
||||
|
||||
@@ -139,11 +138,6 @@ jobs:
|
||||
with:
|
||||
toolchain: nightly
|
||||
|
||||
- name: Should run benchmarks with all precisions
|
||||
if: inputs.all_precisions
|
||||
run: |
|
||||
echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Select HPU board
|
||||
run: |
|
||||
echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}"
|
||||
@@ -156,11 +150,12 @@ jobs:
|
||||
run: |
|
||||
echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}"
|
||||
make pull_hpu_files
|
||||
make BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
|
||||
make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
|
||||
env:
|
||||
OP_FLAVOR: ${{ matrix.op_flavor }}
|
||||
BENCH_TYPE: ${{ matrix.bench_type }}
|
||||
BENCH_COMMAND: ${{ matrix.command }}
|
||||
PRECISIONS_SET: ${{ inputs.precisions_set }}
|
||||
|
||||
- name: Parse results
|
||||
run: |
|
||||
|
||||
80
.github/workflows/generate_svg_common.yml
vendored
80
.github/workflows/generate_svg_common.yml
vendored
@@ -1,17 +1,85 @@
|
||||
name: generate_svg_common.yml
|
||||
name: generate_svg_common
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
backend:
|
||||
type: string
|
||||
required: true
|
||||
hardware_name:
|
||||
type: string
|
||||
required: true
|
||||
layer:
|
||||
type: string
|
||||
required: true
|
||||
pbs_kind: # Valid values are 'classical', 'multi_bit' or 'any'
|
||||
type: string
|
||||
required: true
|
||||
grouping_factor: # Valid values are 2, 3, or 4
|
||||
type: string
|
||||
default: 4
|
||||
bench_type: # Valid values are 'latency', 'throughput'
|
||||
type: string
|
||||
required: true
|
||||
time_span_days:
|
||||
type: string
|
||||
default: 60
|
||||
output_filename:
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER:
|
||||
required: true
|
||||
DATA_EXTRACTOR_DATABASE_HOST:
|
||||
required: true
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD:
|
||||
required: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
# zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency
|
||||
|
||||
jobs:
|
||||
placeholder:
|
||||
name: generate_svg_common/placeholder
|
||||
generate-table:
|
||||
name: generate_svg_common/generate-table
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- run: |
|
||||
echo "Hello this is a placeholder workflow"
|
||||
- name: Checkout tfhe-rs
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
with:
|
||||
persist-credentials: 'false'
|
||||
|
||||
- name: Produce table from database
|
||||
run: |
|
||||
python3 -m pip install -r ci/data_extractor/requirements.txt
|
||||
python3 ci/data_extractor/src/data_extractor.py "${OUTPUT_FILENAME}" \
|
||||
--generate-svg \
|
||||
--branch "${REF_NAME}" \
|
||||
--backend "${BACKEND}" \
|
||||
--hardware "${HARDWARE_NAME}" \
|
||||
--tfhe-rs-layer "${LAYER}" \
|
||||
--pbs-kind "${PBS_KIND}" \
|
||||
--grouping-factor "${GROUPING_FACTOR}" \
|
||||
--bench-type "${BENCH_TYPE}" \
|
||||
--time-span-days "${TIME_SPAN}"
|
||||
env:
|
||||
OUTPUT_FILENAME: ${{ inputs.output_filename }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
BACKEND: ${{ inputs.backend }}
|
||||
HARDWARE_NAME: ${{ inputs.hardware_name }}
|
||||
LAYER: ${{ inputs.layer }}
|
||||
PBS_KIND: ${{ inputs.pbs_kind }}
|
||||
GROUPING_FACTOR: ${{ inputs.grouping_factor }}
|
||||
BENCH_TYPE: ${{ inputs.bench_type }}
|
||||
TIME_SPAN: ${{ inputs.time_span_days }}
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
- name: Upload tables
|
||||
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
|
||||
with:
|
||||
name: ${{ github.sha }}_${{ inputs.backend }}_${{ inputs.layer }}_${{ inputs.pbs_kind }}_${{ inputs.bench_type }}_tables
|
||||
# This will upload all the file generated
|
||||
path: ${{ inputs.output_filename }}*.svg
|
||||
retention-days: 60
|
||||
|
||||
175
.github/workflows/generate_svgs.yml
vendored
175
.github/workflows/generate_svgs.yml
vendored
@@ -1,17 +1,178 @@
|
||||
name: generate_svgs.yml
|
||||
# Generate benchmark SVGs for public documentation
|
||||
name: generate_documentation_svgs
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
time_span_days:
|
||||
type: string
|
||||
required: true
|
||||
generate-cpu-svgs:
|
||||
type: boolean
|
||||
default: true
|
||||
generate-gpu-svgs:
|
||||
type: boolean
|
||||
default: true
|
||||
generate-hpu-svgs:
|
||||
type: boolean
|
||||
default: true
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER:
|
||||
required: true
|
||||
DATA_EXTRACTOR_DATABASE_HOST:
|
||||
required: true
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD:
|
||||
required: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
# zizmor: ignore[concurrency-limits] caller workflow is responsible for the concurrency
|
||||
|
||||
jobs:
|
||||
placeholder:
|
||||
name: generate_svgs/placeholder
|
||||
runs-on: ubuntu-latest
|
||||
# -----------------------------------------------------------
|
||||
# Integer benchmarks tables
|
||||
# -----------------------------------------------------------
|
||||
|
||||
steps:
|
||||
- run: |
|
||||
echo "Hello this is a placeholder workflow"
|
||||
cpu-integer-latency-table:
|
||||
name: generate_documentation_svgs/cpu-integer-latency-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-cpu-svgs
|
||||
with:
|
||||
backend: cpu
|
||||
hardware_name: hpc7a.96xlarge
|
||||
layer: integer
|
||||
pbs_kind: classical
|
||||
bench_type: latency
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: cpu-integer-benchmark-tuniform-2m128-latency
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
cpu-integer-throughput-table:
|
||||
name: generate_documentation_svgs/cpu-integer-latency-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-cpu-svgs
|
||||
with:
|
||||
backend: cpu
|
||||
hardware_name: hpc7a.96xlarge
|
||||
layer: integer
|
||||
pbs_kind: classical
|
||||
bench_type: throughput
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: cpu-integer-benchmark-tuniform-2m128-throughput
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
gpu-integer-latency-table:
|
||||
name: generate_documentation_svgs/gpu-integer-latency-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-gpu-svgs
|
||||
with:
|
||||
backend: gpu
|
||||
hardware_name: n3-H100-SXM5x8
|
||||
layer: integer
|
||||
pbs_kind: multi_bit
|
||||
grouping_factor: 4
|
||||
bench_type: latency
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-latency
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
gpu-integer-throughput-table:
|
||||
name: generate_documentation_svgs/gpu-integer-throughput-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-gpu-svgs
|
||||
with:
|
||||
backend: gpu
|
||||
hardware_name: n3-H100-SXM5x8
|
||||
layer: integer
|
||||
pbs_kind: multi_bit
|
||||
grouping_factor: 4
|
||||
bench_type: throughput
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: gpu-integer-benchmark-h100x8-sxm5-multi-bit-tuniform-2m128-throughput
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
hpu-integer-latency-table:
|
||||
name: generate_documentation_svgs/hpu-integer-latency-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-hpu-svgs
|
||||
with:
|
||||
backend: hpu
|
||||
hardware_name: hpu_x1
|
||||
layer: integer
|
||||
pbs_kind: classical
|
||||
bench_type: latency
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-latency
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
hpu-integer-throughput-table:
|
||||
name: generate_documentation_svgs/hpu-integer-throughput-table
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-hpu-svgs
|
||||
with:
|
||||
backend: hpu
|
||||
hardware_name: hpu_x1
|
||||
layer: integer
|
||||
pbs_kind: classical
|
||||
bench_type: throughput
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: hpu-integer-benchmark-hpux1-tuniform-2m128-throughput
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
# -----------------------------------------------------------
|
||||
# PBS benchmarks tables
|
||||
# -----------------------------------------------------------
|
||||
|
||||
cpu-pbs-tables:
|
||||
name: generate_documentation_svgs/cpu-pbs-tables
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-cpu-svgs
|
||||
with:
|
||||
backend: cpu
|
||||
hardware_name: hpc7a.96xlarge
|
||||
layer: core_crypto
|
||||
pbs_kind: any
|
||||
grouping_factor: 4
|
||||
bench_type: latency
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: cpu-pbs-benchmark
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
gpu-pbs-tables:
|
||||
name: generate_documentation_svgs/gpu-pbs-tables
|
||||
uses: ./.github/workflows/generate_svg_common.yml
|
||||
if: inputs.generate-gpu-svgs
|
||||
with:
|
||||
backend: gpu
|
||||
hardware_name: n3-H100-SXM5x8
|
||||
layer: core_crypto
|
||||
pbs_kind: any
|
||||
grouping_factor: 4
|
||||
bench_type: latency
|
||||
time_span_days: ${{ inputs.time_span_days }}
|
||||
output_filename: gpu-pbs-benchmark
|
||||
secrets:
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATA_EXTRACTOR_DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATA_EXTRACTOR_DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATA_EXTRACTOR_DATABASE_PASSWORD }}
|
||||
|
||||
22
Makefile
22
Makefile
@@ -15,7 +15,7 @@ GEN_KEY_CACHE_MULTI_BIT_ONLY?=FALSE
|
||||
GEN_KEY_CACHE_COVERAGE_ONLY?=FALSE
|
||||
PARSE_INTEGER_BENCH_CSV_FILE?=tfhe_rs_integer_benches.csv
|
||||
FAST_TESTS?=FALSE
|
||||
FAST_BENCH?=FALSE
|
||||
BIT_SIZES_SET?=ALL
|
||||
NIGHTLY_TESTS?=FALSE
|
||||
BENCH_OP_FLAVOR?=DEFAULT
|
||||
BENCH_TYPE?=latency
|
||||
@@ -1353,28 +1353,28 @@ print_doc_bench_parameters:
|
||||
|
||||
.PHONY: bench_integer # Run benchmarks for unsigned integer
|
||||
bench_integer: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_signed_integer # Run benchmarks for signed integer
|
||||
bench_signed_integer: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend
|
||||
bench_integer_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
|
||||
|
||||
.PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend
|
||||
bench_signed_integer_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
|
||||
@@ -1383,7 +1383,7 @@ bench_signed_integer_gpu: install_rs_check_toolchain
|
||||
bench_integer_hpu: install_rs_check_toolchain
|
||||
source ./setup_hpu.sh --config $(HPU_CONFIG); \
|
||||
export V80_PCIE_DEV=${V80_PCIE_DEV}; \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick
|
||||
@@ -1433,7 +1433,7 @@ bench_integer_aes256_gpu: install_rs_check_toolchain
|
||||
.PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters
|
||||
bench_integer_multi_bit: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
@@ -1441,7 +1441,7 @@ bench_integer_multi_bit: install_rs_check_toolchain
|
||||
.PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters
|
||||
bench_signed_integer_multi_bit: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
@@ -1449,7 +1449,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain
|
||||
.PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters
|
||||
bench_integer_multi_bit_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
|
||||
@@ -1457,7 +1457,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain
|
||||
.PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters
|
||||
bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --profile release_lto_off --
|
||||
@@ -1513,7 +1513,7 @@ bench_pbs: install_rs_check_toolchain
|
||||
|
||||
.PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend
|
||||
bench_pbs_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_BENCH_BIT_SIZES_SET=$(BIT_SIZES_SET) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench core_crypto-pbs \
|
||||
--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark --profile release_lto_off
|
||||
|
||||
@@ -351,12 +351,33 @@ const BENCH_BIT_SIZES: [usize; 8] = [2, 8, 16, 32, 40, 64, 128, 256];
|
||||
const BENCH_BIT_SIZES: [usize; 7] = [8, 16, 32, 40, 64, 128, 256];
|
||||
const HPU_BENCH_BIT_SIZES: [usize; 5] = [8, 16, 32, 64, 128];
|
||||
const MULTI_BIT_CPU_SIZES: [usize; 5] = [8, 16, 32, 40, 64];
|
||||
const BENCH_BIT_SIZES_DOCUMENTATION: [usize; 5] = [8, 16, 32, 64, 128];
|
||||
|
||||
#[derive(Default)]
|
||||
pub enum BitSizesSet {
|
||||
#[default]
|
||||
Fast,
|
||||
All,
|
||||
Documentation,
|
||||
}
|
||||
|
||||
impl BitSizesSet {
|
||||
pub fn from_env() -> Result<Self, String> {
|
||||
let raw_value = env::var("__TFHE_RS_BENCH_BIT_SIZES_SET").unwrap_or("fast".to_string());
|
||||
match raw_value.to_lowercase().as_str() {
|
||||
"fast" => Ok(BitSizesSet::Fast),
|
||||
"all" => Ok(BitSizesSet::All),
|
||||
"documentation" => Ok(BitSizesSet::Documentation),
|
||||
_ => Err(format!("bit sizes set '{raw_value}' is not supported")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// User configuration in which benchmarks must be run.
|
||||
#[derive(Default)]
|
||||
pub struct EnvConfig {
|
||||
pub is_multi_bit: bool,
|
||||
pub is_fast_bench: bool,
|
||||
pub bit_sizes_set: BitSizesSet,
|
||||
}
|
||||
|
||||
impl EnvConfig {
|
||||
@@ -366,22 +387,21 @@ impl EnvConfig {
|
||||
ParamType::MultiBit | ParamType::MultiBitDocumentation
|
||||
);
|
||||
|
||||
let is_fast_bench = match env::var("__TFHE_RS_FAST_BENCH") {
|
||||
Ok(val) => val.to_lowercase() == "true",
|
||||
Err(_) => false,
|
||||
};
|
||||
|
||||
EnvConfig {
|
||||
is_multi_bit,
|
||||
is_fast_bench,
|
||||
bit_sizes_set: BitSizesSet::from_env().unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get precisions values to benchmark.
|
||||
pub fn bit_sizes(&self) -> Vec<usize> {
|
||||
if self.is_fast_bench {
|
||||
FAST_BENCH_BIT_SIZES.to_vec()
|
||||
} else if self.is_multi_bit {
|
||||
let bit_sizes_set = match self.bit_sizes_set {
|
||||
BitSizesSet::Fast => return FAST_BENCH_BIT_SIZES.to_vec(),
|
||||
BitSizesSet::All => BENCH_BIT_SIZES.to_vec(),
|
||||
BitSizesSet::Documentation => return BENCH_BIT_SIZES_DOCUMENTATION.to_vec(),
|
||||
};
|
||||
|
||||
if self.is_multi_bit {
|
||||
if cfg!(feature = "gpu") {
|
||||
BENCH_BIT_SIZES.to_vec()
|
||||
} else {
|
||||
@@ -390,7 +410,7 @@ impl EnvConfig {
|
||||
} else if cfg!(feature = "hpu") {
|
||||
HPU_BENCH_BIT_SIZES.to_vec()
|
||||
} else {
|
||||
BENCH_BIT_SIZES.to_vec()
|
||||
bit_sizes_set
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user