mirror of
https://github.com/zama-ai/tfhe-rs.git
synced 2026-01-08 14:23:53 -05:00
400 lines
16 KiB
YAML
400 lines
16 KiB
YAML
# Run performance regression benchmarks and return parsed results to associated pull-request.
|
|
name: benchmark_perf_regression
|
|
|
|
on:
|
|
issue_comment:
|
|
types: [ created ]
|
|
pull_request:
|
|
types: [ labeled ]
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
|
|
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
RUST_BACKTRACE: "full"
|
|
RUST_MIN_STACK: "8388608"
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
permissions: { }
|
|
|
|
# zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow
|
|
|
|
jobs:
|
|
verify-triggering-actor:
|
|
name: benchmark_perf_regression/verify-actor
|
|
if: (github.event_name == 'pull_request' &&
|
|
(contains(github.event.label.name, 'bench-perfs-cpu') ||
|
|
contains(github.event.label.name, 'bench-perfs-gpu'))) ||
|
|
(github.event.issue.pull_request && startsWith(github.event.comment.body, '/bench'))
|
|
uses: ./.github/workflows/verify_triggering_actor.yml
|
|
secrets:
|
|
ALLOWED_TEAM: ${{ secrets.RELEASE_TEAM }}
|
|
READ_ORG_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
|
|
|
|
prepare-benchmarks:
|
|
name: benchmark_perf_regression/prepare-benchmarks
|
|
needs: verify-triggering-actor
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
commands: ${{ steps.set_commands.outputs.commands }}
|
|
slab-backend: ${{ steps.set_slab_details.outputs.backend }}
|
|
slab-profile: ${{ steps.set_slab_details.outputs.profile }}
|
|
hardware-name: ${{ steps.get_hardware_name.outputs.name }}
|
|
tfhe-backend: ${{ steps.set_regression_details.outputs.tfhe-backend }}
|
|
selected-regression-profile: ${{ steps.set_regression_details.outputs.selected-profile }}
|
|
custom-env: ${{ steps.get_custom_env.outputs.custom_env }}
|
|
permissions:
|
|
pull-requests: write # Needed to write a comment in a pull-request
|
|
steps:
|
|
- name: Checkout tfhe-rs repo
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
|
with:
|
|
persist-credentials: 'false'
|
|
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
|
|
|
- name: Acknowledge issue comment
|
|
if: github.event_name == 'issue_comment'
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
|
|
with:
|
|
comment-id: ${{ github.event.comment.id }}
|
|
reactions: '+1'
|
|
|
|
- name: Display workflow run URL
|
|
if: github.event_name == 'issue_comment'
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
|
|
with:
|
|
issue-number: ${{ github.event.issue.number }}
|
|
body: |
|
|
User triggered performance regression benchmark.
|
|
Workflow run URL: ${{ env.ACTION_RUN_URL }}
|
|
|
|
- name: Generate CPU benchmarks command from label
|
|
if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-cpu'))
|
|
run: |
|
|
echo "DEFAULT_BENCH_OPTIONS=--backend cpu" >> "${GITHUB_ENV}"
|
|
|
|
- name: Generate GPU benchmarks command from label
|
|
if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-gpu'))
|
|
run: |
|
|
echo "DEFAULT_BENCH_OPTIONS=--backend gpu" >> "${GITHUB_ENV}"
|
|
|
|
# TODO add support for HPU backend
|
|
|
|
- name: Install Python requirements
|
|
run: |
|
|
python3 -m pip install -r ci/perf_regression/requirements.txt
|
|
|
|
- name: Generate cargo commands and env from label
|
|
if: github.event_name == 'pull_request'
|
|
run: |
|
|
python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}"
|
|
echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
|
|
|
- name: Dump issue comment into file # To avoid possible code-injection
|
|
if: github.event_name == 'issue_comment'
|
|
run: |
|
|
echo "${COMMENT_BODY}" >> dumped_comment.txt
|
|
env:
|
|
COMMENT_BODY: ${{ github.event.comment.body }}
|
|
|
|
- name: Generate cargo commands and env
|
|
if: github.event_name == 'issue_comment'
|
|
run: |
|
|
python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)"
|
|
echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
|
|
|
- name: Set commands output
|
|
id: set_commands
|
|
run: | # zizmor: ignore[template-injection] this env variable is safe
|
|
echo "commands=${{ toJSON(env.COMMANDS) }}" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Set Slab details outputs
|
|
id: set_slab_details
|
|
run: |
|
|
echo "backend=$(cat ci/perf_regression/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}"
|
|
echo "profile=$(cat ci/perf_regression/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Get hardware name
|
|
id: get_hardware_name
|
|
run: | # zizmor: ignore[template-injection] these interpolations are safe
|
|
HARDWARE_NAME=$(python3 ci/hardware_finder.py "${{ steps.set_slab_details.outputs.backend }}" "${{ steps.set_slab_details.outputs.profile }}");
|
|
echo "name=${HARDWARE_NAME}" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Set regression details outputs
|
|
id: set_regression_details
|
|
run: |
|
|
echo "tfhe-backend=$(cat ci/perf_regression/perf_regression_tfhe_rs_backend_config.txt)" >> "${GITHUB_OUTPUT}"
|
|
echo "selected-profile=$(cat ci/perf_regression/perf_regression_selected_profile_config.txt)" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Get custom env vars
|
|
id: get_custom_env
|
|
run: |
|
|
echo "custom_env=$(cat ci/perf_regression/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}"
|
|
|
|
setup-instance:
|
|
name: benchmark_perf_regression/setup-instance
|
|
needs: prepare-benchmarks
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
runner-name: ${{ steps.start-instance.outputs.label }}
|
|
steps:
|
|
- name: Start instance
|
|
id: start-instance
|
|
uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2
|
|
with:
|
|
mode: start
|
|
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
|
|
slab-url: ${{ secrets.SLAB_BASE_URL }}
|
|
job-secret: ${{ secrets.JOB_SECRET }}
|
|
backend: ${{ needs.prepare-benchmarks.outputs.slab-backend }}
|
|
profile: ${{ needs.prepare-benchmarks.outputs.slab-profile }}
|
|
|
|
install-cuda-dependencies-if-required:
|
|
name: benchmark_perf_regression/install-cuda-dependencies-if-required
|
|
needs: [ prepare-benchmarks, setup-instance ]
|
|
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
|
|
strategy:
|
|
matrix:
|
|
# explicit include-based build matrix, of known valid options
|
|
include:
|
|
- cuda: "12.8"
|
|
gcc: 11
|
|
steps:
|
|
- name: Checkout tfhe-rs repo
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
|
with:
|
|
persist-credentials: 'false'
|
|
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
|
|
|
- name: Setup Hyperstack dependencies
|
|
if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack'
|
|
uses: ./.github/actions/gpu_setup
|
|
with:
|
|
cuda-version: ${{ matrix.cuda }}
|
|
gcc-version: ${{ matrix.gcc }}
|
|
|
|
regression-benchmarks:
|
|
name: benchmark_perf_regression/regression-benchmarks
|
|
needs: [ prepare-benchmarks, setup-instance, install-cuda-dependencies-if-required ]
|
|
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
|
|
concurrency:
|
|
group: ${{ github.workflow_ref }}_${{ needs.prepare-benchmarks.outputs.slab-backend }}_${{ needs.prepare-benchmarks.outputs.slab-profile }}
|
|
cancel-in-progress: true
|
|
timeout-minutes: 720 # 12 hours
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 1
|
|
matrix:
|
|
command: ${{ fromJson(needs.prepare-benchmarks.outputs.commands) }}
|
|
steps:
|
|
- name: Checkout tfhe-rs repo
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
|
with:
|
|
fetch-depth: 0 # Needed to get commit hash
|
|
persist-credentials: 'false'
|
|
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
|
|
|
- name: Get benchmark details
|
|
run: |
|
|
COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}");
|
|
{
|
|
echo "BENCH_DATE=$(date --iso-8601=seconds)";
|
|
echo "COMMIT_DATE=${COMMIT_DATE}";
|
|
echo "COMMIT_HASH=$(git describe --tags --dirty)";
|
|
} >> "${GITHUB_ENV}"
|
|
env:
|
|
SHA: ${{ github.sha }}
|
|
|
|
- name: Export custom env variables
|
|
run: | # zizmor: ignore[template-injection] this env variable is safe
|
|
{
|
|
${{ needs.prepare-benchmarks.outputs.custom-env }}
|
|
} >> "$GITHUB_ENV"
|
|
|
|
# Re-export environment variables as dependencies setup perform this task in the previous job.
|
|
# Local env variables are cleaned at the end of each job.
|
|
- name: Export CUDA variables
|
|
if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack'
|
|
shell: bash
|
|
run: |
|
|
echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}"
|
|
echo "PATH=$PATH:$CUDA_PATH/bin" >> "${GITHUB_PATH}"
|
|
echo "LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}"
|
|
echo "CUDA_MODULE_LOADER=EAGER" >> "${GITHUB_ENV}"
|
|
env:
|
|
CUDA_PATH: /usr/local/cuda-12.8
|
|
|
|
- name: Export gcc and g++ variables
|
|
if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack'
|
|
shell: bash
|
|
run: |
|
|
{
|
|
echo "CC=/usr/bin/gcc-${GCC_VERSION}";
|
|
echo "CXX=/usr/bin/g++-${GCC_VERSION}";
|
|
echo "CUDAHOSTCXX=/usr/bin/g++-${GCC_VERSION}";
|
|
} >> "${GITHUB_ENV}"
|
|
env:
|
|
GCC_VERSION: 11
|
|
|
|
- name: Install rust
|
|
uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # zizmor: ignore[stale-action-refs] this action doesn't create releases
|
|
with:
|
|
toolchain: nightly
|
|
|
|
- name: Checkout Slab repo
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
|
with:
|
|
repository: zama-ai/slab
|
|
path: slab
|
|
persist-credentials: 'false'
|
|
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
|
|
|
- name: Run regression benchmarks
|
|
run: |
|
|
make BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom
|
|
env:
|
|
BENCH_COMMAND: ${{ matrix.command }}
|
|
|
|
- name: Parse results
|
|
run: |
|
|
python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \
|
|
--database tfhe_rs \
|
|
--hardware "${HARDWARE_NAME}" \
|
|
--backend "${TFHE_BACKEND}" \
|
|
--project-version "${COMMIT_HASH}" \
|
|
--branch "${REF_NAME}" \
|
|
--commit-date "${COMMIT_DATE}" \
|
|
--bench-date "${BENCH_DATE}" \
|
|
--walk-subdirs \
|
|
--name-suffix regression \
|
|
--bench-type "${BENCH_TYPE}"
|
|
|
|
echo "RESULTS_FILE_SHA=$(sha256sum "${RESULTS_FILENAME}" | cut -d " " -f1)" >> "${GITHUB_ENV}"
|
|
env:
|
|
HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
|
|
TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
|
|
REF_NAME: ${{ github.head_ref || github.ref_name }}
|
|
BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }}
|
|
|
|
- name: Upload parsed results artifact
|
|
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4
|
|
with:
|
|
name: ${{ github.sha }}_regression_${{ env.RESULTS_FILE_SHA }} # RESULT_FILE_SHA is needed to avoid collision between matrix.command runs
|
|
path: ${{ env.RESULTS_FILENAME }}
|
|
|
|
- name: Send data to Slab
|
|
shell: bash
|
|
run: |
|
|
python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \
|
|
--slab-url "${SLAB_URL}"
|
|
env:
|
|
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
|
SLAB_URL: ${{ secrets.SLAB_URL }}
|
|
|
|
check-regressions:
|
|
name: benchmark_perf_regression/check-regressions
|
|
needs: [ prepare-benchmarks, regression-benchmarks ]
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
pull-requests: write # Needed to write a comment in a pull-request
|
|
contents: read # Needed to set up Python dependencies
|
|
env:
|
|
REF_NAME: ${{ github.head_ref || github.ref_name }}
|
|
steps:
|
|
- name: Checkout tfhe-rs repo
|
|
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
|
|
with:
|
|
persist-credentials: 'false'
|
|
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
|
|
|
- name: Install recent Python
|
|
uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
|
with:
|
|
python-version: '3.12'
|
|
pip-install: -r ci/data_extractor/requirements.txt -r ci/perf_regression/requirements.txt
|
|
|
|
- name: Fetch data
|
|
run: |
|
|
python3 ci/data_extractor/src/data_extractor.py regression_data \
|
|
--generate-regression-json \
|
|
--regression-profiles ci/regression.toml \
|
|
--regression-selected-profile "${REGRESSION_PROFILE}" \
|
|
--backend "${TFHE_BACKEND}" \
|
|
--hardware "${HARDWARE_NAME}" \
|
|
--branch "${REF_NAME}" \
|
|
--time-span-days 60
|
|
env:
|
|
REGRESSION_PROFILE: ${{ needs.prepare-benchmarks.outputs.selected-regression-profile }}
|
|
TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
|
|
HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
|
|
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATABASE_HOST }}
|
|
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATABASE_USER }}
|
|
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
|
|
|
|
- name: Generate regression report
|
|
run: |
|
|
python3 ci/perf_regression/perf_regression.py check_regression \
|
|
--results-file regression_data.json \
|
|
--generate-report
|
|
|
|
- name: Write report in pull-request
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
|
|
with:
|
|
issue-number: ${{ github.event.pull_request.number || github.event.issue.number }}
|
|
body-path: ci/perf_regression/regression_report.md
|
|
|
|
comment-on-failure:
|
|
name: benchmark_perf_regression/comment-on-failure
|
|
needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ failure() && github.event_name == 'issue_comment' }}
|
|
continue-on-error: true
|
|
permissions:
|
|
pull-requests: write # Needed to write a comment in a pull-request
|
|
steps:
|
|
- name: Write failure message
|
|
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
|
|
with:
|
|
issue-number: ${{ github.event.issue.number }}
|
|
body: |
|
|
:x: Performance regression benchmark failed ([workflow run](${{ env.ACTION_RUN_URL }}))
|
|
|
|
slack-notify:
|
|
name: benchmark_perf_regression/slack-notify
|
|
needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ failure() }}
|
|
continue-on-error: true
|
|
steps:
|
|
- name: Send message
|
|
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
|
|
env:
|
|
SLACK_COLOR: failure
|
|
SLACK_MESSAGE: "Performance regression benchmarks failed. (${{ env.ACTION_RUN_URL }})"
|
|
|
|
teardown-instance:
|
|
name: benchmark_perf_regression/teardown-instance
|
|
if: ${{ always() && needs.setup-instance.result == 'success' }}
|
|
needs: [ setup-instance, regression-benchmarks ]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Stop instance
|
|
id: stop-instance
|
|
uses: zama-ai/slab-github-runner@973c1d22702de8d0acd2b34e83404c96ed92c264 # v1.4.2
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
|
|
slab-url: ${{ secrets.SLAB_BASE_URL }}
|
|
job-secret: ${{ secrets.JOB_SECRET }}
|
|
label: ${{ needs.setup-instance.outputs.runner-name }}
|
|
|
|
- name: Slack Notification
|
|
if: ${{ failure() }}
|
|
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
|
|
env:
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "Instance teardown (regression-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
|