mirror of
https://github.com/zama-ai/tfhe-rs.git
synced 2026-01-07 22:04:10 -05:00
chore(ci): check for performance regression and create report
After running performances regression benchmarks, a performance changes checking is executed. It will fetch results data with an external tool then it will look for anomaly in changes. Finally it will produce a report as an issue comment with any anomaly display in a Markdown array. A folded section of the report message contains all the results from the benchmark. Note that a fully custom benchmark triggered from an issue comment would not generate a report. In addition HPU performance regression benchmark is not supported yet.
This commit is contained in:
David Testé
138
.github/workflows/benchmark_perf_regression.yml
vendored
138
.github/workflows/benchmark_perf_regression.yml
vendored
@@ -41,7 +41,12 @@ jobs:
|
||||
slab-backend: ${{ steps.set_slab_details.outputs.backend }}
|
||||
slab-profile: ${{ steps.set_slab_details.outputs.profile }}
|
||||
hardware-name: ${{ steps.get_hardware_name.outputs.name }}
|
||||
tfhe-backend: ${{ steps.set_regression_details.outputs.tfhe-backend }}
|
||||
selected-regression-profile: ${{ steps.set_regression_details.outputs.selected-profile }}
|
||||
custom-env: ${{ steps.get_custom_env.outputs.custom_env }}
|
||||
permissions:
|
||||
# Needed to react to benchmark command in issue comment
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout tfhe-rs repo
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
@@ -49,23 +54,27 @@ jobs:
|
||||
persist-credentials: 'false'
|
||||
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
|
||||
- name: Generate cpu benchmarks command from label
|
||||
- name: Generate CPU benchmarks command from label
|
||||
if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-cpu'))
|
||||
run: |
|
||||
echo "DEFAULT_BENCH_OPTIONS=--backend cpu" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Generate cpu benchmarks command from label
|
||||
- name: Generate GPU benchmarks command from label
|
||||
if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-gpu'))
|
||||
run: |
|
||||
echo "DEFAULT_BENCH_OPTIONS=--backend gpu" >> "${GITHUB_ENV}"
|
||||
|
||||
# TODO add support for HPU backend
|
||||
|
||||
- name: Install Python requirements
|
||||
run: |
|
||||
python3 -m pip install -r ci/perf_regression/requirements.txt
|
||||
|
||||
- name: Generate cargo commands and env from label
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
python3 ci/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}"
|
||||
echo "COMMANDS=$(cat ci/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
||||
python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}"
|
||||
echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Dump issue comment into file # To avoid possible code-injection
|
||||
if: github.event_name == 'issue_comment'
|
||||
@@ -77,8 +86,15 @@ jobs:
|
||||
- name: Generate cargo commands and env
|
||||
if: github.event_name == 'issue_comment'
|
||||
run: |
|
||||
python3 ci/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)"
|
||||
echo "COMMANDS=$(cat ci/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
||||
python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)"
|
||||
echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Acknowledge issue comment
|
||||
if: github.event_name == 'issue_comment'
|
||||
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
|
||||
with:
|
||||
comment-id: ${{ github.event.comment.id }}
|
||||
reactions: +1
|
||||
|
||||
- name: Set commands output
|
||||
id: set_commands
|
||||
@@ -88,8 +104,8 @@ jobs:
|
||||
- name: Set Slab details outputs
|
||||
id: set_slab_details
|
||||
run: |
|
||||
echo "backend=$(cat ci/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
echo "profile=$(cat ci/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
echo "backend=$(cat ci/perf_regression/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
echo "profile=$(cat ci/perf_regression/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Get hardware name
|
||||
id: get_hardware_name
|
||||
@@ -97,10 +113,16 @@ jobs:
|
||||
HARDWARE_NAME=$(python3 ci/hardware_finder.py "${{ steps.set_slab_details.outputs.backend }}" "${{ steps.set_slab_details.outputs.profile }}");
|
||||
echo "name=${HARDWARE_NAME}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Set regression details outputs
|
||||
id: set_regression_details
|
||||
run: |
|
||||
echo "tfhe-backend=$(cat ci/perf_regression/perf_regression_tfhe_rs_backend_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
echo "selected-profile=$(cat ci/perf_regression/perf_regression_selected_profile_config.txt)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Get custom env vars
|
||||
id: get_custom_env
|
||||
run: |
|
||||
echo "custom_env=$(cat ci/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}"
|
||||
echo "custom_env=$(cat ci/perf_regression/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
setup-instance:
|
||||
name: benchmark_perf_regression/setup-instance
|
||||
@@ -134,7 +156,6 @@ jobs:
|
||||
- name: Checkout tfhe-rs repo
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: 'false'
|
||||
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
|
||||
@@ -154,14 +175,15 @@ jobs:
|
||||
cancel-in-progress: true
|
||||
timeout-minutes: 720 # 12 hours
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
command: ${{ fromJson(needs.prepare-benchmarks.outputs.commands) }}
|
||||
steps:
|
||||
- name: Checkout tfhe-rs repo with tags
|
||||
- name: Checkout tfhe-rs repo
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-depth: 0 # Needed to get commit hash
|
||||
persist-credentials: 'false'
|
||||
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
|
||||
@@ -222,7 +244,7 @@ jobs:
|
||||
|
||||
- name: Run regression benchmarks
|
||||
run: |
|
||||
make BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom
|
||||
make BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom
|
||||
env:
|
||||
BENCH_COMMAND: ${{ matrix.command }}
|
||||
|
||||
@@ -231,6 +253,7 @@ jobs:
|
||||
python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \
|
||||
--database tfhe_rs \
|
||||
--hardware "${HARDWARE_NAME}" \
|
||||
--backend "${TFHE_BACKEND}" \
|
||||
--project-version "${COMMIT_HASH}" \
|
||||
--branch "${REF_NAME}" \
|
||||
--commit-date "${COMMIT_DATE}" \
|
||||
@@ -238,15 +261,18 @@ jobs:
|
||||
--walk-subdirs \
|
||||
--name-suffix regression \
|
||||
--bench-type "${BENCH_TYPE}"
|
||||
|
||||
echo "RESULTS_FILE_SHA=$(sha256sum "${RESULTS_FILENAME}" | cut -d " " -f1)" >> "${GITHUB_ENV}"
|
||||
env:
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }}
|
||||
HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
|
||||
TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
|
||||
REF_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }}
|
||||
|
||||
- name: Upload parsed results artifact
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
|
||||
with:
|
||||
name: ${{ github.sha }}_regression
|
||||
name: ${{ github.sha }}_regression_${{ env.RESULTS_FILE_SHA }} # RESULT_FILE_SHA is needed to avoid collision between matrix.command runs
|
||||
path: ${{ env.RESULTS_FILENAME }}
|
||||
|
||||
- name: Send data to Slab
|
||||
@@ -258,9 +284,81 @@ jobs:
|
||||
JOB_SECRET: ${{ secrets.JOB_SECRET }}
|
||||
SLAB_URL: ${{ secrets.SLAB_URL }}
|
||||
|
||||
check-regressions:
|
||||
name: benchmark_perf_regression/check-regressions
|
||||
needs: [ prepare-benchmarks, regression-benchmarks ]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
# Needed to write a comment in a pull-request
|
||||
pull-requests: write
|
||||
# Needed to set up Python dependencies
|
||||
contents: read
|
||||
env:
|
||||
REF_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
steps:
|
||||
- name: Checkout tfhe-rs repo
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
with:
|
||||
persist-credentials: 'false'
|
||||
token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
|
||||
|
||||
- name: Install recent Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Fetch data
|
||||
run: |
|
||||
python3 -m pip install -r ci/data_extractor/requirements.txt
|
||||
python3 ci/data_extractor/src/data_extractor.py regression_data \
|
||||
--generate-regression-json \
|
||||
--regression-profiles ci/regression.toml \
|
||||
--regression-selected-profile "${REGRESSION_PROFILE}" \
|
||||
--backend "${TFHE_BACKEND}" \
|
||||
--hardware "${HARDWARE_NAME}" \
|
||||
--branch "${REF_NAME}" \
|
||||
--time-span-days 60
|
||||
env:
|
||||
REGRESSION_PROFILE: ${{ needs.prepare-benchmarks.outputs.selected-regression-profile }}
|
||||
TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
|
||||
HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
|
||||
DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATABASE_HOST }}
|
||||
DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATABASE_USER }}
|
||||
DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
|
||||
|
||||
- name: Generate regression report
|
||||
run: |
|
||||
python3 -m pip install -r ci/perf_regression/requirements.txt
|
||||
python3 ci/perf_regression/perf_regression.py check_regression \
|
||||
--results-file regression_data.json \
|
||||
--generate-report
|
||||
|
||||
- name: Write report in pull-request
|
||||
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
body-path: ci/perf_regression/regression_report.md
|
||||
|
||||
comment-on-failure:
|
||||
name: benchmark_perf_regression/comment-on-failure
|
||||
needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ failure() && github.event_name == 'issue_comment' }}
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
# Needed to write a comment in a pull-request
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Write failure message
|
||||
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
|
||||
with:
|
||||
issue-number: ${{ github.event.issue.number }}
|
||||
body: |
|
||||
:x: Performance regression benchmark failed ([workflow run](${{ env.ACTION_RUN_URL }}))
|
||||
|
||||
slack-notify:
|
||||
name: benchmark_perf_regression/slack-notify
|
||||
needs: [ prepare-benchmarks, setup-instance, regression-benchmarks ]
|
||||
needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ failure() }}
|
||||
continue-on-error: true
|
||||
@@ -268,10 +366,8 @@ jobs:
|
||||
- name: Send message
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
|
||||
env:
|
||||
SLACK_COLOR: ${{ needs.regression-benchmarks.result }}
|
||||
SLACK_MESSAGE: "Performance regression benchmarks finished with status: ${{ needs.regression-benchmarks.result }}. (${{ env.ACTION_RUN_URL }})"
|
||||
|
||||
# TODO Add job for regression calculation
|
||||
SLACK_COLOR: failure
|
||||
SLACK_MESSAGE: "Performance regression benchmarks failed. (${{ env.ACTION_RUN_URL }})"
|
||||
|
||||
teardown-instance:
|
||||
name: benchmark_perf_regression/teardown-instance
|
||||
|
||||
42
Makefile
42
Makefile
@@ -1336,28 +1336,28 @@ print_doc_bench_parameters:
|
||||
bench_integer: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_signed_integer # Run benchmarks for signed integer
|
||||
bench_signed_integer: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed-bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend
|
||||
bench_integer_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-bench \
|
||||
--bench integer \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend
|
||||
bench_signed_integer_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed-bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_hpu # Run benchmarks for integer on HPU backend
|
||||
@@ -1366,28 +1366,28 @@ bench_integer_hpu: install_rs_check_toolchain
|
||||
export V80_PCIE_DEV=${V80_PCIE_DEV}; \
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick
|
||||
|
||||
.PHONY: bench_integer_compression # Run benchmarks for unsigned integer compression
|
||||
bench_integer_compression: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench glwe_packing_compression-integer-bench \
|
||||
--bench integer-glwe_packing_compression \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_compression_gpu
|
||||
bench_integer_compression_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench glwe_packing_compression-integer-bench \
|
||||
--bench integer-glwe_packing_compression \
|
||||
--features=integer,internal-keycache,gpu,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_zk_gpu
|
||||
bench_integer_zk_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench zk-pke-bench \
|
||||
--bench integer-zk-pke \
|
||||
--features=integer,internal-keycache,gpu,pbs-stats,zk-pok -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters
|
||||
@@ -1395,7 +1395,7 @@ bench_integer_multi_bit: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-bench \
|
||||
--bench integer \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters
|
||||
@@ -1403,7 +1403,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed-bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters
|
||||
@@ -1411,7 +1411,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-bench \
|
||||
--bench integer \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters
|
||||
@@ -1419,14 +1419,14 @@ bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
|
||||
__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench integer-signed-bench \
|
||||
--bench integer-signed \
|
||||
--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
|
||||
|
||||
.PHONY: bench_integer_zk # Run benchmarks for integer encryption with ZK proofs
|
||||
bench_integer_zk: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench zk-pke-bench \
|
||||
--bench integer-zk-pke \
|
||||
--features=integer,internal-keycache,zk-pok,nightly-avx512,pbs-stats \
|
||||
-p tfhe-benchmark --
|
||||
|
||||
@@ -1454,56 +1454,56 @@ bench_boolean: install_rs_check_toolchain
|
||||
bench_ks: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench ks-bench \
|
||||
--bench core_crypto-ks \
|
||||
--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_ks_gpu # Run benchmarks for keyswitch on GPU backend
|
||||
bench_ks_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench ks-bench \
|
||||
--bench core_crypto-ks \
|
||||
--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_pbs # Run benchmarks for PBS
|
||||
bench_pbs: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench pbs-bench \
|
||||
--bench core_crypto-pbs \
|
||||
--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend
|
||||
bench_pbs_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench pbs-bench \
|
||||
--bench core_crypto-pbs \
|
||||
--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_ks_pbs # Run benchmarks for KS-PBS
|
||||
bench_ks_pbs: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench ks-pbs-bench \
|
||||
--bench core_crypto-ks-pbs \
|
||||
--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_ks_pbs_gpu # Run benchmarks for KS-PBS on GPU backend
|
||||
bench_ks_pbs_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench ks-pbs-bench \
|
||||
--bench core_crypto-ks-pbs \
|
||||
--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_pbs128 # Run benchmarks for PBS using FFT 128 bits
|
||||
bench_pbs128: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench pbs128-bench \
|
||||
--bench core_crypto-pbs128 \
|
||||
--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
.PHONY: bench_pbs128_gpu # Run benchmarks for PBS using FFT 128 bits on GPU
|
||||
bench_pbs128_gpu: install_rs_check_toolchain
|
||||
RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
|
||||
cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
|
||||
--bench pbs128-bench \
|
||||
--bench core_crypto-pbs128 \
|
||||
--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
|
||||
|
||||
bench_web_js_api_parallel_chrome: browser_path = "$(WEB_RUNNER_DIR)/chrome/chrome-linux64/chrome"
|
||||
|
||||
4
ci/data_extractor/requirements.txt
Normal file
4
ci/data_extractor/requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
psycopg2-binary==2.9.9
|
||||
py-markdown-table==1.3.0
|
||||
svg.py==1.5.0
|
||||
black
|
||||
490
ci/data_extractor/src/benchmark_specs.py
Normal file
490
ci/data_extractor/src/benchmark_specs.py
Normal file
@@ -0,0 +1,490 @@
|
||||
import enum
|
||||
|
||||
from exceptions import ParametersFormatNotSupported
|
||||
|
||||
|
||||
class Backend(enum.StrEnum):
|
||||
"""
|
||||
Represents different types of computation backends used in tfhe-rs.
|
||||
"""
|
||||
|
||||
CPU = "cpu"
|
||||
GPU = "gpu"
|
||||
|
||||
@staticmethod
|
||||
def from_str(backend_name):
|
||||
match backend_name.lower():
|
||||
case "cpu":
|
||||
return Backend.CPU
|
||||
case "gpu":
|
||||
return Backend.GPU
|
||||
case _:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class Layer(enum.StrEnum):
|
||||
"""
|
||||
Represents different types of layers used in tfhe-rs.
|
||||
"""
|
||||
|
||||
HLApi = "hlapi"
|
||||
Integer = "integer"
|
||||
Shortint = "shortint"
|
||||
CoreCrypto = "core_crypto"
|
||||
|
||||
@staticmethod
|
||||
def from_str(layer_name):
|
||||
match layer_name.lower():
|
||||
case "hlapi":
|
||||
return Layer.HLApi
|
||||
case "integer":
|
||||
return Layer.Integer
|
||||
case "shortint":
|
||||
return Layer.Shortint
|
||||
case "core_crypto":
|
||||
return Layer.CoreCrypto
|
||||
case _:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RustType(enum.Enum):
|
||||
"""
|
||||
Represents different integer Rust types used in tfhe-rs.
|
||||
"""
|
||||
|
||||
FheUint2 = 2
|
||||
FheUint4 = 4
|
||||
FheUint8 = 8
|
||||
FheUint16 = 16
|
||||
FheUint32 = 32
|
||||
FheUint64 = 64
|
||||
FheUint128 = 128
|
||||
FheUint256 = 256
|
||||
|
||||
|
||||
ALL_RUST_TYPES = [
|
||||
RustType.FheUint2,
|
||||
RustType.FheUint4,
|
||||
RustType.FheUint8,
|
||||
RustType.FheUint16,
|
||||
RustType.FheUint32,
|
||||
RustType.FheUint64,
|
||||
RustType.FheUint128,
|
||||
RustType.FheUint256,
|
||||
]
|
||||
|
||||
|
||||
class CoreCryptoOperation(enum.StrEnum):
|
||||
"""
|
||||
Represents different core crypto operations performed in tfhe-rs.
|
||||
The values are the ones displayed in the public benchmarks documentation.
|
||||
"""
|
||||
|
||||
KeySwitch = "KS"
|
||||
PBS = "PBS"
|
||||
MultiBitPBS = "MB-PBS"
|
||||
KeyswitchPBS = "KS - PBS"
|
||||
KeySwitchMultiBitPBS = "KS - MB-PBS"
|
||||
|
||||
@staticmethod
|
||||
def from_str(operation_name):
|
||||
match operation_name.lower():
|
||||
case "keyswitch":
|
||||
return CoreCryptoOperation.KeySwitch
|
||||
case "pbs_mem_optimized":
|
||||
return CoreCryptoOperation.PBS
|
||||
case "multi_bit_pbs" | "multi_bit_deterministic_pbs":
|
||||
return CoreCryptoOperation.MultiBitPBS
|
||||
case "ks_pbs":
|
||||
return CoreCryptoOperation.KeyswitchPBS
|
||||
case "multi_bit_ks_pbs" | "multi_bit_deterministic_ks_pbs":
|
||||
return CoreCryptoOperation.KeySwitchMultiBitPBS
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"core crypto operation '{operation_name}' not supported yet"
|
||||
)
|
||||
|
||||
def display_name(self):
|
||||
"""
|
||||
Return the human-friendly name recorded for a given operation.
|
||||
This name is parameter-independent.
|
||||
|
||||
:return: The name as recorded by tfhe-benchmark crate
|
||||
:rtype: str
|
||||
"""
|
||||
match self:
|
||||
case CoreCryptoOperation.KeySwitch:
|
||||
return "ks"
|
||||
case CoreCryptoOperation.PBS:
|
||||
return "pbs"
|
||||
case CoreCryptoOperation.MultiBitPBS:
|
||||
return "pbs"
|
||||
case CoreCryptoOperation.KeyswitchPBS:
|
||||
return "ks-pbs"
|
||||
case CoreCryptoOperation.KeySwitchMultiBitPBS:
|
||||
return "ks-pbs"
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"display name for {self} not implemented yet"
|
||||
)
|
||||
|
||||
|
||||
class SignFlavor(enum.StrEnum):
|
||||
"""
|
||||
Represents the sign of integer benchmarks.
|
||||
"""
|
||||
|
||||
Signed = "signed"
|
||||
Unsigned = "unsigned"
|
||||
|
||||
|
||||
class OperandType(enum.StrEnum):
|
||||
"""
|
||||
Represents the type of operand use in a benchmark.
|
||||
Ciphertext means encrypted-encrypted operation.
|
||||
PlainText means encrypted-plaintext operation.
|
||||
"""
|
||||
|
||||
CipherText = "CipherText" # As represented in the database
|
||||
PlainText = "PlainText"
|
||||
|
||||
|
||||
class PBSKind(enum.StrEnum):
|
||||
"""
|
||||
Represents the kind of parameter set used for Programmable Bootstrapping operation.
|
||||
"""
|
||||
|
||||
Classical = "classical"
|
||||
MultiBit = "multi_bit"
|
||||
Any = "any" # Special variant used when user doesn't care about the PBS kind
|
||||
|
||||
@staticmethod
|
||||
def from_str(pbs_name):
|
||||
match pbs_name.lower():
|
||||
case "classical":
|
||||
return PBSKind.Classical
|
||||
case "multi_bit":
|
||||
return PBSKind.MultiBit
|
||||
case "any":
|
||||
return PBSKind.Any
|
||||
case _:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class NoiseDistribution(enum.StrEnum):
|
||||
"""
|
||||
Represents the noise distribution used in the parameter set.
|
||||
"""
|
||||
|
||||
Gaussian = "gaussian"
|
||||
TUniform = "tuniform"
|
||||
|
||||
@staticmethod
|
||||
def from_str(distrib_name):
|
||||
match distrib_name.lower():
|
||||
case "gaussian":
|
||||
return NoiseDistribution.Gaussian
|
||||
case "tuniform":
|
||||
return NoiseDistribution.TUniform
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"noise distribution '{distrib_name}' not supported yet"
|
||||
)
|
||||
|
||||
|
||||
class ErrorFailureProbability(enum.IntEnum):
|
||||
"""
|
||||
Represents the error failure probability associated with a parameter set.
|
||||
"""
|
||||
|
||||
TWO_MINUS_40 = 40
|
||||
TWO_MINUS_64 = 64
|
||||
TWO_MINUS_128 = 128
|
||||
|
||||
@staticmethod
|
||||
def from_param_name(name):
|
||||
parts = name.split("_")
|
||||
for part in parts:
|
||||
if not part.startswith("2M"):
|
||||
continue
|
||||
|
||||
match int(part.lstrip("2M")):
|
||||
case 40:
|
||||
return ErrorFailureProbability.TWO_MINUS_40
|
||||
case 64:
|
||||
return ErrorFailureProbability.TWO_MINUS_64
|
||||
case 128:
|
||||
return ErrorFailureProbability.TWO_MINUS_128
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"error failure probability '{part}' not supported yet"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Could not find p-fail value in '{name}'")
|
||||
|
||||
def to_str(self):
|
||||
match self:
|
||||
case ErrorFailureProbability.TWO_MINUS_40:
|
||||
return "2M40"
|
||||
case ErrorFailureProbability.TWO_MINUS_64:
|
||||
return "2M64"
|
||||
case ErrorFailureProbability.TWO_MINUS_128:
|
||||
return "2M128"
|
||||
case _:
|
||||
raise ValueError(
|
||||
f"error failure probability str conversion '{self}' not supported yet"
|
||||
)
|
||||
|
||||
|
||||
class ParamsDefinition:
|
||||
"""
|
||||
Represents a parameter definition for specific cryptographic settings.
|
||||
|
||||
The class `ParamsDefinition` is designed to parse and manage parameters derived from
|
||||
a specified parameter name. It facilitates comparison, representation, and hashing of
|
||||
parameter configurations. Parameters related to message size, carry size, noise characteristics,
|
||||
failure probabilities, and other functionalities are extracted and stored for streamlined
|
||||
usage across the system.
|
||||
|
||||
:param param_name: The raw name of the parameter set.
|
||||
:type param_name: str
|
||||
"""
|
||||
|
||||
def __init__(self, param_name: str):
|
||||
self.message_size = None
|
||||
self.carry_size = None
|
||||
self.pbs_kind = None
|
||||
self.grouping_factor = None
|
||||
self.noise_distribution = None
|
||||
self.atomic_pattern = None
|
||||
self.p_fail = None
|
||||
self.version = None
|
||||
self.details = {}
|
||||
|
||||
self._parse_param_name(param_name)
|
||||
|
||||
def __eq__(self, other):
|
||||
return (
|
||||
self.message_size == other.message_size
|
||||
and self.carry_size == other.carry_size
|
||||
and self.pbs_kind == other.pbs_kind
|
||||
and self.grouping_factor == other.grouping_factor
|
||||
and self.noise_distribution == other.noise_distribution
|
||||
and self.atomic_pattern == other.atomic_pattern
|
||||
and self.p_fail == other.p_fail
|
||||
and self.version == other.version
|
||||
and self.details == other.details
|
||||
)
|
||||
|
||||
def __lt__(self, other):
|
||||
|
||||
return (
|
||||
self.message_size < other.message_size
|
||||
and self.carry_size < other.carry_size
|
||||
and self.p_fail < other.p_fail
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(
|
||||
(
|
||||
self.message_size,
|
||||
self.carry_size,
|
||||
self.pbs_kind,
|
||||
self.grouping_factor,
|
||||
self.noise_distribution,
|
||||
self.atomic_pattern,
|
||||
self.p_fail,
|
||||
self.version,
|
||||
)
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"ParamsDefinition(message_size={self.message_size}, carry_size={self.carry_size}, pbs_kind={self.pbs_kind}, grouping_factor={self.grouping_factor}, noise_distribution={self.noise_distribution}, atomic_pattern={self.atomic_pattern}, p_fail={self.p_fail}, version={self.version}, details={self.details})"
|
||||
|
||||
def _parse_param_name(self, param_name: str) -> None:
|
||||
split_params = param_name.split("_")
|
||||
|
||||
if split_params[0].startswith("V"):
|
||||
minor_version = split_params.pop(1)
|
||||
major_version = split_params.pop(0).strip("V")
|
||||
self.version = major_version + "_" + minor_version
|
||||
|
||||
# Use to know if a parameter set is a derivative of a compute one (e.g. compression parameters)
|
||||
params_variation_parts = []
|
||||
for part in split_params:
|
||||
if part == "PARAM":
|
||||
self.details["variation"] = "_".join(params_variation_parts)
|
||||
break
|
||||
|
||||
params_variation_parts.append(part)
|
||||
|
||||
try:
|
||||
self.p_fail = ErrorFailureProbability.from_param_name(param_name)
|
||||
pfail_index = split_params.index(self.p_fail.to_str())
|
||||
except ValueError or NotImplementedError:
|
||||
# Default error probability may not be shown in the name
|
||||
self.p_fail = ErrorFailureProbability.TWO_MINUS_128
|
||||
pfail_index = None
|
||||
|
||||
if pfail_index:
|
||||
noise_distribution_index = pfail_index - 1
|
||||
self.noise_distribution = NoiseDistribution.from_str(
|
||||
split_params[noise_distribution_index]
|
||||
)
|
||||
else:
|
||||
# Default noise distribution may not be shown in the name
|
||||
self.noise_distribution = NoiseDistribution.TUniform
|
||||
noise_distribution_index = None
|
||||
|
||||
try:
|
||||
self.message_size = int(split_params[split_params.index("MESSAGE") + 1])
|
||||
carry_size_index = split_params.index("CARRY") + 1
|
||||
self.carry_size = int(split_params[carry_size_index])
|
||||
self.atomic_pattern = "_".join(
|
||||
split_params[carry_size_index + 1 : noise_distribution_index]
|
||||
)
|
||||
except ValueError:
|
||||
# Might be a Boolean parameters set
|
||||
raise ParametersFormatNotSupported(param_name)
|
||||
|
||||
try:
|
||||
if noise_distribution_index:
|
||||
self.atomic_pattern = "_".join(
|
||||
split_params[carry_size_index + 1 : noise_distribution_index]
|
||||
)
|
||||
else:
|
||||
self.atomic_pattern = "_".join(split_params[carry_size_index + 1 :])
|
||||
except ValueError:
|
||||
# Might be a Boolean parameters set
|
||||
raise ParametersFormatNotSupported(param_name)
|
||||
|
||||
try:
|
||||
self.details["trailing_details"] = "_".join(split_params[pfail_index + 1 :])
|
||||
except IndexError:
|
||||
# No trailing details
|
||||
pass
|
||||
|
||||
try:
|
||||
# This is a multi-bit parameters set
|
||||
self.grouping_factor = int(split_params[split_params.index("GROUP") + 1])
|
||||
self.pbs_kind = PBSKind.MultiBit
|
||||
except ValueError:
|
||||
# This is a classical parameters set
|
||||
self.pbs_kind = PBSKind.Classical
|
||||
|
||||
|
||||
class BenchDetails:
|
||||
"""
|
||||
Represents the details of a benchmark test for different layers.
|
||||
|
||||
This class is designed to parse benchmark information, extract meaningful
|
||||
details such as operation name, parameters, and relevant configuration
|
||||
based on the layer type. It allows for comparison between different benchmark
|
||||
details and outputs structured information for representation or hashing.
|
||||
|
||||
:param layer: The layer the benchmark pertains to.
|
||||
:type layer: Layer
|
||||
:param bench_full_name: Complete name of the benchmark operation.
|
||||
:type bench_full_name: str
|
||||
:param bit_size: The bit size associated with the benchmark.
|
||||
:type bit_size: int
|
||||
"""
|
||||
|
||||
def __init__(self, layer: Layer, bench_full_name: str, bit_size: int):
|
||||
self.layer = layer
|
||||
|
||||
self.operation_name = None
|
||||
self.bit_size = bit_size
|
||||
self.params = None
|
||||
# Only relevant for Integer layer
|
||||
self.sign_flavor = None
|
||||
# Only relevant for HLApi layer
|
||||
self.rust_type = None
|
||||
|
||||
self.parse_test_name(bench_full_name)
|
||||
|
||||
def __repr__(self):
|
||||
return f"BenchDetails(layer={self.layer.value}, operation_name={self.operation_name}, bit_size={self.bit_size}, params={self.params}, {self.sign_flavor})"
|
||||
|
||||
def __str__(self):
|
||||
return self.__repr__()
|
||||
|
||||
def __eq__(self, other):
|
||||
return (
|
||||
self.layer == other.layer
|
||||
and self.operation_name == other.operation_name
|
||||
and self.bit_size == other.bit_size
|
||||
and self.params == other.params
|
||||
and self.sign_flavor == other.sign_flavor
|
||||
and self.rust_type == other.rust_type
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(
|
||||
(
|
||||
self.layer,
|
||||
self.operation_name,
|
||||
self.bit_size,
|
||||
self.params,
|
||||
self.rust_type,
|
||||
self.sign_flavor,
|
||||
)
|
||||
)
|
||||
|
||||
def parse_test_name(self, name) -> None:
|
||||
"""
|
||||
Parse test name to split relevant parts.
|
||||
|
||||
:param name: The raw test name.
|
||||
:type name: str
|
||||
|
||||
:return: None
|
||||
"""
|
||||
parts = name.split("::")
|
||||
|
||||
for part in parts:
|
||||
if "PARAM" in part:
|
||||
self.params = part.partition("_mean")[0]
|
||||
break
|
||||
|
||||
match self.layer:
|
||||
case Layer.Integer:
|
||||
op_name_index = 2 if parts[1] == "cuda" else 1
|
||||
if parts[op_name_index] == "signed":
|
||||
op_name_index += 1
|
||||
self.sign_flavor = SignFlavor.Signed
|
||||
self.operation_name = parts[op_name_index]
|
||||
elif parts[op_name_index] == "unsigned":
|
||||
# This is a pattern used by benchmark run on CUDA
|
||||
op_name_index += 1
|
||||
self.sign_flavor = SignFlavor.Unsigned
|
||||
self.operation_name = parts[op_name_index]
|
||||
else:
|
||||
self.sign_flavor = SignFlavor.Unsigned
|
||||
self.operation_name = parts[op_name_index]
|
||||
self.params = parts[op_name_index + 1]
|
||||
if "compression" in parts[op_name_index]:
|
||||
self.rust_type = "_".join(
|
||||
(parts[op_name_index], parts[-1].split("_")[0])
|
||||
)
|
||||
case Layer.CoreCrypto:
|
||||
self.operation_name = parts[2] if parts[1] == "cuda" else parts[1]
|
||||
case Layer.HLApi:
|
||||
if parts[1] == "cuda":
|
||||
self.operation_name = "::".join(parts[2:-1])
|
||||
else:
|
||||
self.operation_name = "::".join(parts[1:-1])
|
||||
self.rust_type = parts[-1].partition("_mean")[0]
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"layer '{self.layer}' not supported yet for name parsing"
|
||||
)
|
||||
|
||||
def get_params_definition(self) -> ParamsDefinition:
|
||||
"""
|
||||
Returns the definition of parameters based on the current instance's parameters.
|
||||
|
||||
:return: A ParamsDefinition object that encapsulates the parameter definition.
|
||||
:rtype: ParamsDefinition
|
||||
"""
|
||||
return ParamsDefinition(self.params)
|
||||
43
ci/data_extractor/src/config.py
Normal file
43
ci/data_extractor/src/config.py
Normal file
@@ -0,0 +1,43 @@
|
||||
import argparse
|
||||
import pathlib
|
||||
|
||||
from benchmark_specs import Backend, Layer, PBSKind
|
||||
|
||||
|
||||
class UserConfig:
|
||||
"""
|
||||
Manages the configuration provided by the user input.
|
||||
|
||||
This class encapsulates the user-provided configuration data necessary for execution.
|
||||
It sets up various attributes based on the input arguments, converting or transforming
|
||||
values as needed to ensure compatibility and correct behavior within the system.
|
||||
|
||||
:param input_args: The input arguments provided by the user.
|
||||
:type input_args: argparse.Namespace
|
||||
"""
|
||||
|
||||
def __init__(self, input_args: argparse.Namespace):
|
||||
self.output_file = input_args.output_file
|
||||
|
||||
self.database = input_args.database
|
||||
self.backend = Backend.from_str(input_args.backend.lower())
|
||||
self.hardware = input_args.hardware # This input is case-sensitive
|
||||
|
||||
self.head_branch = input_args.branch.lower()
|
||||
self.base_branch = input_args.base_branch.lower()
|
||||
|
||||
self.project_version = input_args.project_version
|
||||
|
||||
self.bench_date = input_args.bench_date
|
||||
self.time_span_days = input_args.time_span_days
|
||||
|
||||
self.layer = Layer.from_str(input_args.layer.lower())
|
||||
self.pbs_kind = PBSKind.from_str(input_args.pbs_kind)
|
||||
self.grouping_factor = input_args.grouping_factor
|
||||
|
||||
self.regression_selected_profile = input_args.regression_selected_profile
|
||||
self.regression_profiles_path = (
|
||||
pathlib.Path(input_args.regression_profiles)
|
||||
if input_args.regression_profiles
|
||||
else None
|
||||
)
|
||||
292
ci/data_extractor/src/connector.py
Normal file
292
ci/data_extractor/src/connector.py
Normal file
@@ -0,0 +1,292 @@
|
||||
import configparser
|
||||
import datetime
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
import psycopg2
|
||||
from benchmark_specs import BenchDetails, Layer, OperandType, PBSKind
|
||||
from config import UserConfig
|
||||
from exceptions import NoDataFound
|
||||
|
||||
|
||||
class PostgreConfig:
|
||||
"""
|
||||
Represents the configuration to connect to a PostgreSQL database.
|
||||
|
||||
This class is designed to manage and load PostgreSQL database credentials
|
||||
from a configuration file and override them with environment variables if available.
|
||||
|
||||
:param path: Path to the configuration file.
|
||||
:type path: str, optional
|
||||
|
||||
:ivar host: Host address of the PostgreSQL database.
|
||||
:type host: str
|
||||
:ivar user: Username for connecting to the PostgreSQL database.
|
||||
:type user: str
|
||||
:ivar password: Password for connecting to the PostgreSQL database.
|
||||
:type password: str
|
||||
"""
|
||||
|
||||
def __init__(self, path: str = None):
|
||||
self.host = None
|
||||
self.user = None
|
||||
self.password = None
|
||||
|
||||
if path:
|
||||
self._from_config_file(path)
|
||||
|
||||
self._override_with_env()
|
||||
|
||||
def _from_config_file(self, path):
|
||||
"""
|
||||
Parse configuration file containing credentials to Postgre database.
|
||||
|
||||
:param path: path to the configuration file as :class:`str`
|
||||
|
||||
:return: parsed file as :class:`configparser.ConfigParser`
|
||||
"""
|
||||
path = pathlib.Path(path)
|
||||
conf = configparser.ConfigParser()
|
||||
conf.read(path)
|
||||
|
||||
for option_name in ("user", "password", "host"):
|
||||
try:
|
||||
attr_value = conf.get("postgre", option_name)
|
||||
except ValueError:
|
||||
print(f"Cannot find mandatory option '{option_name}' in config file")
|
||||
raise
|
||||
|
||||
setattr(self, option_name, attr_value)
|
||||
|
||||
def _override_with_env(self):
|
||||
self.host = os.environ.get("DATA_EXTRACTOR_DATABASE_HOST", self.host)
|
||||
self.user = os.environ.get("DATA_EXTRACTOR_DATABASE_USER", self.user)
|
||||
self.password = os.environ.get(
|
||||
"DATA_EXTRACTOR_DATABASE_PASSWORD", self.password
|
||||
)
|
||||
|
||||
|
||||
class PostgreConnector:
|
||||
def __init__(self, conf: PostgreConfig):
|
||||
"""
|
||||
Initializes the class with a configuration object.
|
||||
|
||||
:param conf: The configuration object used to set up
|
||||
the connection or relevant settings.
|
||||
:type conf: PostgreConfig
|
||||
"""
|
||||
self.conf = conf
|
||||
self._conn = None
|
||||
|
||||
def __del__(self):
|
||||
if self._conn:
|
||||
# Avoid dandling connection on server side.
|
||||
self._conn.close()
|
||||
|
||||
def connect_to_database(self, dbname: str):
|
||||
"""
|
||||
Create a connection to a Postgre instance.
|
||||
|
||||
:param dbname: Name of the database to connect to.
|
||||
:type dbname: str
|
||||
"""
|
||||
print("Connecting to database... ", end="", flush=True)
|
||||
start_date = datetime.datetime.now()
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(
|
||||
dbname=dbname,
|
||||
user=self.conf.user,
|
||||
password=self.conf.password,
|
||||
host=self.conf.host,
|
||||
)
|
||||
except psycopg2.Error as err:
|
||||
print(f"Failed to connect to Postgre database")
|
||||
raise
|
||||
else:
|
||||
elapsed = (datetime.datetime.now() - start_date).total_seconds()
|
||||
print(f"connected in {elapsed}s")
|
||||
|
||||
self._conn = conn
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
Close the connection to the database.
|
||||
|
||||
:return: None
|
||||
"""
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
def fetch_benchmark_data(
|
||||
self,
|
||||
user_config: UserConfig,
|
||||
operand_type: OperandType = None,
|
||||
operation_filter: list = None,
|
||||
layer: Layer = None,
|
||||
branch: str = None,
|
||||
name_suffix: str = "_mean_avx512",
|
||||
last_value_only: bool = True,
|
||||
) -> dict[BenchDetails, list[int]]:
|
||||
"""
|
||||
Fetches benchmark data from the database based on various filtering criteria.
|
||||
|
||||
Filters and constructs a query based on the provided user configuration and optional
|
||||
parameters such as operand type, operation filters, layer, and others. It retrieves
|
||||
specific benchmark metrics like test names, bit sizes, metric values, and their last
|
||||
inserted values. The data is fetched for a given hardware, backend, branch, and project
|
||||
version, or within a computed time range.
|
||||
|
||||
:param user_config: User's configuration specifying backend, hardware, branch, layer,
|
||||
project version, PBS kind, benchmark date, and time span.
|
||||
:type user_config: UserConfig
|
||||
:param operand_type: Optional operand type filter.
|
||||
:type operand_type: OperandType, optional
|
||||
:param operation_filter: Optional list of operation name filters for partial test name matching.
|
||||
:type operation_filter: list, optional
|
||||
:param layer: Optional tfhe-rs layer filter
|
||||
:type layer: Layer, optional
|
||||
:param branch: Optional branch filter, defaulting to the user's head branch if not specified.
|
||||
:type branch: str, optional
|
||||
:param name_suffix: Suffix to match the test names, defaulting to "_mean_avx512".
|
||||
:type name_suffix: str, optional
|
||||
:param last_value_only: A flag indicating whether to fetch only the most recent metric value for each benchmark.
|
||||
:type last_value_only: bool
|
||||
|
||||
:return: Fetched benchmark data filtered and formatted as per the query.
|
||||
:rtype: dict[BenchDetails, list[int]]
|
||||
"""
|
||||
backend = user_config.backend
|
||||
branch = branch.lower() if branch else user_config.head_branch
|
||||
hardware = user_config.hardware
|
||||
layer = layer if layer else user_config.layer
|
||||
version = user_config.project_version
|
||||
pbs_kind = user_config.pbs_kind
|
||||
|
||||
timestamp_range_end = user_config.bench_date
|
||||
timestamp = datetime.datetime.fromisoformat(timestamp_range_end)
|
||||
time_span_delta = datetime.timedelta(days=user_config.time_span_days)
|
||||
timestamp_range_start = datetime.datetime.isoformat(timestamp - time_span_delta)
|
||||
if layer == Layer.CoreCrypto:
|
||||
# Before this date we used another name format that is not parsable by this tool.
|
||||
cutoff_date = "2024-02-28T00:00:00"
|
||||
timestamp_range_start = max(cutoff_date, timestamp_range_start)
|
||||
|
||||
filters = list()
|
||||
|
||||
filters.append(f"h.name = '{hardware}'")
|
||||
filters.append(f"bk.name = '{backend}'")
|
||||
filters.append(f"b.name = '{branch}'")
|
||||
|
||||
name_suffix = f"\\{name_suffix}"
|
||||
if backend == "cpu":
|
||||
filters.append(f"test.name LIKE '{layer}::%{name_suffix}'")
|
||||
elif backend == "gpu":
|
||||
filters.append(f"test.name LIKE '{layer}::cuda::%{name_suffix}'")
|
||||
|
||||
if version:
|
||||
filters.append(f"pv.name = '{version}'")
|
||||
else:
|
||||
filters.append(f"m.insert_time >= '{timestamp_range_start}'")
|
||||
filters.append(f"m.insert_time <= '{timestamp_range_end}'")
|
||||
|
||||
# First iteration to fetch only default operations
|
||||
filters.append("test.name NOT SIMILAR TO '%(smart|unchecked)_%'")
|
||||
match pbs_kind:
|
||||
case PBSKind.Classical:
|
||||
filters.append(
|
||||
"p.crypto_parameters_alias NOT SIMILAR TO '%_MULTI_BIT_%'"
|
||||
)
|
||||
case PBSKind.MultiBit:
|
||||
filters.append("p.crypto_parameters_alias SIMILAR TO '%_MULTI_BIT_%'")
|
||||
case PBSKind.Any:
|
||||
# No need to add a filter
|
||||
pass
|
||||
|
||||
if operand_type:
|
||||
filters.append(f"p.operand_type = '{operand_type.value}'")
|
||||
|
||||
if operation_filter:
|
||||
conditions = [
|
||||
f"test.name LIKE '%::{op_name}::%'" for op_name in operation_filter
|
||||
]
|
||||
filters.append("({})".format(" OR ".join(conditions)))
|
||||
|
||||
# Throughput is not supported yet
|
||||
filters.append("test.name NOT SIMILAR TO '%::throughput::%'")
|
||||
|
||||
select_parts = (
|
||||
"SELECT",
|
||||
"test.name as test,",
|
||||
"p.bit_size as bit_size,",
|
||||
"m.value as value,",
|
||||
"m.insert_time,",
|
||||
"LAST_VALUE (value)",
|
||||
"OVER (PARTITION BY test.name ORDER BY m.insert_time RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value",
|
||||
"FROM benchmark.metrics as m",
|
||||
"LEFT JOIN benchmark.project_version as pv ON m.project_version_id = pv.id",
|
||||
"LEFT JOIN benchmark.hardware as h ON m.hardware_id = h.id",
|
||||
"LEFT JOIN benchmark.backend as bk ON m.backend_id = bk.id",
|
||||
"LEFT JOIN benchmark.branch as b ON b.id = m.branch_id",
|
||||
"LEFT JOIN benchmark.test as test ON test.id = m.test_id",
|
||||
"LEFT JOIN benchmark.parameters as p ON p.id = m.parameters_id",
|
||||
)
|
||||
|
||||
sql_string = format(
|
||||
"{} WHERE {} GROUP BY test, bit_size, value, m.insert_time ORDER BY m.insert_time DESC"
|
||||
).format(" ".join(select_parts), " AND ".join(filters))
|
||||
|
||||
return self._fetch_data(
|
||||
sql_string,
|
||||
version,
|
||||
timestamp_range_start,
|
||||
timestamp_range_end,
|
||||
hardware,
|
||||
layer,
|
||||
last_value_only,
|
||||
)
|
||||
|
||||
def _fetch_data(
|
||||
self,
|
||||
sql_string,
|
||||
version,
|
||||
timestamp_range_start,
|
||||
timestamp_range_end,
|
||||
hw,
|
||||
layer: Layer,
|
||||
last_value_only: bool,
|
||||
) -> dict[BenchDetails, list[int]]:
|
||||
with self._conn.cursor() as curs:
|
||||
start = datetime.datetime.now()
|
||||
print(f"Fetching data (hardware: {hw}, layer: {layer.value})...", end="")
|
||||
|
||||
curs.execute(sql_string)
|
||||
lines = curs.fetchall()
|
||||
|
||||
end = datetime.datetime.now()
|
||||
print(f"done in {(end - start).total_seconds()}s")
|
||||
|
||||
results = dict()
|
||||
|
||||
if not lines:
|
||||
if version:
|
||||
msg = f"no data found under commit hash '{version}'"
|
||||
else:
|
||||
msg = f"no data found in date range [{timestamp_range_start}, {timestamp_range_end}]"
|
||||
raise NoDataFound(msg)
|
||||
|
||||
for line in lines:
|
||||
bit_width = line[1]
|
||||
|
||||
bench_details = BenchDetails(layer, line[0], bit_width)
|
||||
value = line[-1] if last_value_only else line[-3]
|
||||
try:
|
||||
timings = results[bench_details]
|
||||
if last_value_only:
|
||||
continue
|
||||
else:
|
||||
timings.append(value)
|
||||
except KeyError:
|
||||
results[bench_details] = [value]
|
||||
|
||||
return results
|
||||
403
ci/data_extractor/src/data_extractor.py
Normal file
403
ci/data_extractor/src/data_extractor.py
Normal file
@@ -0,0 +1,403 @@
|
||||
"""
|
||||
data_extractor
|
||||
--------------
|
||||
|
||||
Extract benchmarks results from Zama PostgreSQL instance.
|
||||
It will output the filtered results in a file formatted as CSV.
|
||||
|
||||
PostgreSQL connection configuration can be passed through a configuration file or via environment variables.
|
||||
When using the environment variables, make sure to set the following ones:
|
||||
* DATA_EXTRACTOR_DATABASE_HOST
|
||||
* DATA_EXTRACTOR_DATABASE_USER
|
||||
* DATA_EXTRACTOR_DATABASE_PASSWORD
|
||||
|
||||
Note that if provided, environment variables will take precedence over the configuration file.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import formatter
|
||||
import sys
|
||||
from formatter import (CSVFormatter, GenericFormatter, MarkdownFormatter,
|
||||
SVGFormatter)
|
||||
|
||||
import config
|
||||
import connector
|
||||
import regression
|
||||
from benchmark_specs import Backend, Layer, OperandType, PBSKind, RustType
|
||||
|
||||
import utils
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
parser.add_argument(
|
||||
"output_file", help="File storing parsed results (with no extension)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c" "--config-file",
|
||||
dest="config_file",
|
||||
help="Location of configuration file containing credentials to connect to "
|
||||
"PostgreSQL instance",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bench-date",
|
||||
dest="bench_date",
|
||||
default=datetime.datetime.now().isoformat(),
|
||||
help=(
|
||||
"Last insertion date to look for in the database,"
|
||||
" formatted as ISO 8601 timestamp YYYY-MM-DDThh:mm:ss"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--database",
|
||||
dest="database",
|
||||
default="tfhe_rs",
|
||||
help="Name of the database used to store results",
|
||||
)
|
||||
group.add_argument(
|
||||
"-w",
|
||||
"--hardware",
|
||||
dest="hardware",
|
||||
default="hpc7a.96xlarge",
|
||||
help="Hardware reference used to perform benchmark",
|
||||
)
|
||||
group.add_argument(
|
||||
"--hardware-comp",
|
||||
dest="hardware_comp",
|
||||
help="Comma separated values of hardware to compare. "
|
||||
"The first value would be chosen as baseline.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-V", "--project-version", dest="project_version", help="Commit hash reference"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--branch",
|
||||
dest="branch",
|
||||
default="main",
|
||||
help="Git branch name on which benchmark was performed",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-branch",
|
||||
dest="base_branch",
|
||||
default="main",
|
||||
help="Git base branch name on which benchmark history can be fetched",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
dest="backend",
|
||||
choices=["cpu", "gpu"],
|
||||
default="cpu",
|
||||
help="Backend on which benchmarks have run",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tfhe-rs-layer",
|
||||
dest="layer",
|
||||
default="integer",
|
||||
help="Layer of the tfhe-rs library to filter against",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pbs-kind",
|
||||
dest="pbs_kind",
|
||||
choices=["classical", "multi_bit", "any"],
|
||||
default="classical",
|
||||
help="Kind of PBS to look for",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--grouping-factor",
|
||||
dest="grouping_factor",
|
||||
type=int,
|
||||
choices=[2, 3, 4],
|
||||
help="Grouping factor used in multi-bit parameters set",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--time-span-days",
|
||||
dest="time_span_days",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Numbers of days prior of `bench_date` we search for results in the database",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--regression-profiles",
|
||||
dest="regression_profiles",
|
||||
help="Path to file containing regression profiles formatted as TOML",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--regression-selected-profile",
|
||||
dest="regression_selected_profile",
|
||||
help="Regression profile to select from the regression profiles file to filter out database results",
|
||||
)
|
||||
exclusive_generation_group = parser.add_mutually_exclusive_group()
|
||||
exclusive_generation_group.add_argument(
|
||||
"--generate-markdown",
|
||||
dest="generate_markdown",
|
||||
action="store_true",
|
||||
help="Generate Markdown array",
|
||||
)
|
||||
exclusive_generation_group.add_argument(
|
||||
"--generate-svg",
|
||||
dest="generate_svg",
|
||||
action="store_true",
|
||||
help="Generate SVG table formatted like ones in tfhe-rs documentation",
|
||||
)
|
||||
exclusive_generation_group.add_argument(
|
||||
"--generate-svg-from-markdown",
|
||||
dest="generate_svg_from_file",
|
||||
help="Generate SVG table formatted like ones in tfhe-rs documentation from a Markdown table",
|
||||
)
|
||||
exclusive_generation_group.add_argument(
|
||||
"--generate-regression-json",
|
||||
dest="generate_regression_json",
|
||||
action="store_true",
|
||||
help="Generate JSON file with regression data with all the results from base branch and the lastest results of the development branch",
|
||||
)
|
||||
|
||||
|
||||
def generate_svg_from_file(
|
||||
user_config: config.UserConfig, layer: Layer, input_file: str
|
||||
):
|
||||
"""
|
||||
Generates an SVG file based on a given formatted array in Markdown file.
|
||||
|
||||
:param user_config: An instance of the UserConfig class, used to manage
|
||||
configuration details like backend, PBS kind, and output file paths.
|
||||
:type user_config: config.UserConfig
|
||||
:param layer: The layer information used in SVG formatting and
|
||||
generation.
|
||||
:type layer: Layer
|
||||
:param input_file: File path of the input Markdown file to be converted to
|
||||
SVG format.
|
||||
:type input_file: str
|
||||
|
||||
:return: None
|
||||
"""
|
||||
utils.write_to_svg(
|
||||
SVGFormatter(
|
||||
layer,
|
||||
user_config.backend,
|
||||
user_config.pbs_kind,
|
||||
).generate_svg_table_from_markdown_file(input_file),
|
||||
user_config.output_file,
|
||||
)
|
||||
|
||||
|
||||
def perform_hardware_comparison(
|
||||
user_config: config.UserConfig,
|
||||
layer: Layer,
|
||||
):
|
||||
"""
|
||||
Perform a hardware comparison by fetching benchmark data, computing
|
||||
comparisons, and generating CSV outputs for each hardware configuration. It
|
||||
outputs both raw data and gain-based analysis for comparison between
|
||||
reference and target hardware.
|
||||
|
||||
:param user_config: An instance of the UserConfig class, used to manage
|
||||
configuration details like backend, PBS kind, and output file paths.
|
||||
:type user_config: config.UserConfig
|
||||
:param layer: The layer object containing specific information required for
|
||||
formatting and processing benchmark data.
|
||||
:type layer: Layer
|
||||
|
||||
:return: None
|
||||
"""
|
||||
results = []
|
||||
|
||||
for hw in hardware_list:
|
||||
try:
|
||||
res = conn.fetch_benchmark_data(user_config, operand_type)
|
||||
except RuntimeError as err:
|
||||
print(f"Failed to fetch benchmark data: {err}")
|
||||
sys.exit(2)
|
||||
|
||||
results.append(res)
|
||||
|
||||
output_filename = "".join(
|
||||
[user_config.output_file, "_", hw, "_", operand_type.lower(), ".csv"]
|
||||
)
|
||||
csv_formatter = CSVFormatter(layer, user_config.backend, user_config.pbs_kind)
|
||||
formatted_data = csv_formatter.format_data(
|
||||
res, utils.convert_value_to_readable_text
|
||||
)
|
||||
utils.write_to_csv(
|
||||
csv_formatter.generate_csv(formatted_data),
|
||||
output_filename,
|
||||
)
|
||||
|
||||
gains_results = formatter.compute_comparisons(*results)
|
||||
reference_hardware = hardware_list[0]
|
||||
for i, hw in enumerate(hardware_list[1:]):
|
||||
output_filename = "".join(
|
||||
[
|
||||
user_config.output_file,
|
||||
"_",
|
||||
operand_type.lower(),
|
||||
"_",
|
||||
reference_hardware,
|
||||
"_",
|
||||
hw,
|
||||
"_gains.csv",
|
||||
]
|
||||
)
|
||||
csv_formatter = CSVFormatter(layer, user_config.backend, user_config.pbs_kind)
|
||||
formatted_data = csv_formatter.format_data(
|
||||
gains_results[i],
|
||||
utils.convert_gain_to_text,
|
||||
)
|
||||
utils.write_to_csv(
|
||||
csv_formatter.generate_csv(formatted_data),
|
||||
output_filename,
|
||||
)
|
||||
|
||||
|
||||
def perform_data_extraction(
|
||||
user_config: config.UserConfig,
|
||||
layer: Layer,
|
||||
operand_type: OperandType,
|
||||
output_filename: str,
|
||||
generate_markdown: bool = False,
|
||||
generate_svg: bool = False,
|
||||
):
|
||||
"""
|
||||
Extracts, formats, and processes benchmark data for a specified operand type and
|
||||
saves the results into various file formats such as CSV, Markdown, or SVG based
|
||||
on user configuration.
|
||||
|
||||
:param user_config: An instance of the UserConfig class, used to manage
|
||||
configuration details like backend, PBS kind, and output file paths.
|
||||
:type user_config: config.UserConfig
|
||||
:param layer: Layer object specifying the granularity and context of the
|
||||
operand processing.
|
||||
:type layer: Layer
|
||||
:param operand_type: Type of operand data for which the benchmarks are
|
||||
extracted and processed.
|
||||
:type operand_type: OperandType
|
||||
:param output_filename: The base filename for the output files where results
|
||||
will be saved.
|
||||
:type output_filename: str
|
||||
:param generate_markdown: Boolean flag indicating whether to generate an
|
||||
output file in Markdown (.md) format.
|
||||
:type generate_markdown: bool
|
||||
:param generate_svg: Boolean flag indicating whether to generate an output
|
||||
file in SVG (.svg) format.
|
||||
:type generate_svg: bool
|
||||
:return: None
|
||||
"""
|
||||
try:
|
||||
res = conn.fetch_benchmark_data(user_config, operand_type)
|
||||
except RuntimeError as err:
|
||||
print(f"Failed to fetch benchmark data: {err}")
|
||||
sys.exit(2)
|
||||
|
||||
generic_formatter = GenericFormatter(
|
||||
layer, user_config.backend, user_config.pbs_kind, user_config.grouping_factor
|
||||
)
|
||||
formatted_results = generic_formatter.format_data(
|
||||
res,
|
||||
utils.convert_value_to_readable_text,
|
||||
)
|
||||
|
||||
file_suffix = f"_{operand_type.lower()}"
|
||||
filename = utils.append_suffix_to_filename(output_filename, file_suffix, ".csv")
|
||||
|
||||
utils.write_to_csv(
|
||||
CSVFormatter(layer, user_config.backend, user_config.pbs_kind).generate_csv(
|
||||
formatted_results
|
||||
),
|
||||
filename,
|
||||
)
|
||||
|
||||
generic_arrays = generic_formatter.generate_array(
|
||||
formatted_results,
|
||||
operand_type,
|
||||
excluded_types=[RustType.FheUint2, RustType.FheUint4, RustType.FheUint256],
|
||||
)
|
||||
|
||||
for array in generic_arrays:
|
||||
metadata_suffix = ""
|
||||
if array.metadata:
|
||||
for key, value in array.metadata.items():
|
||||
metadata_suffix += f"_{key}_{value}"
|
||||
|
||||
current_suffix = file_suffix + metadata_suffix
|
||||
|
||||
if generate_markdown:
|
||||
filename = utils.append_suffix_to_filename(
|
||||
output_filename, current_suffix, ".md"
|
||||
)
|
||||
|
||||
data_formatter = MarkdownFormatter(
|
||||
layer, user_config.backend, user_config.pbs_kind
|
||||
)
|
||||
|
||||
utils.write_to_markdown(
|
||||
data_formatter.generate_markdown_array(array),
|
||||
filename,
|
||||
)
|
||||
elif generate_svg:
|
||||
filename = utils.append_suffix_to_filename(
|
||||
output_filename, current_suffix, ".svg"
|
||||
)
|
||||
|
||||
data_formatter = SVGFormatter(
|
||||
layer, user_config.backend, user_config.pbs_kind
|
||||
)
|
||||
|
||||
utils.write_to_svg(
|
||||
data_formatter.generate_svg_table(
|
||||
array,
|
||||
),
|
||||
filename,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
user_config = config.UserConfig(args)
|
||||
layer = user_config.layer
|
||||
|
||||
if args.generate_svg_from_file:
|
||||
generate_svg_from_file(user_config, layer, args.generate_svg_from_file)
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
postgre_config = connector.PostgreConfig(args.config_file)
|
||||
conn = connector.PostgreConnector(postgre_config)
|
||||
conn.connect_to_database(user_config.database)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
|
||||
if args.generate_regression_json:
|
||||
try:
|
||||
regression.perform_regression_json_generation(conn, user_config)
|
||||
except RuntimeError as err:
|
||||
print(f"Failed to perform performance regression JSON: {err}")
|
||||
sys.exit(2)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
hardware_list = (
|
||||
args.hardware_comp.lower().split(",") if args.hardware_comp else None
|
||||
)
|
||||
|
||||
for operand_type in (OperandType.CipherText, OperandType.PlainText):
|
||||
if hardware_list:
|
||||
perform_hardware_comparison(user_config, layer)
|
||||
|
||||
if args.generate_markdown:
|
||||
print("Markdown generation is not supported with comparisons")
|
||||
continue
|
||||
|
||||
if layer == Layer.CoreCrypto and operand_type == OperandType.PlainText:
|
||||
continue
|
||||
|
||||
perform_data_extraction(
|
||||
user_config,
|
||||
layer,
|
||||
operand_type,
|
||||
user_config.output_file,
|
||||
generate_markdown=args.generate_markdown,
|
||||
generate_svg=args.generate_svg,
|
||||
)
|
||||
|
||||
conn.close()
|
||||
20
ci/data_extractor/src/exceptions.py
Normal file
20
ci/data_extractor/src/exceptions.py
Normal file
@@ -0,0 +1,20 @@
|
||||
class NoDataFound(RuntimeError):
|
||||
"""
|
||||
Indicates that no data was found when an operation or search was performed.
|
||||
|
||||
This exception should be raised to signal that a requested operation could not
|
||||
complete because the required data was unavailable. It is typically used in
|
||||
cases where returning an empty result might not be appropriate, and an
|
||||
explicit notice of failure is required.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ParametersFormatNotSupported(Exception):
|
||||
"""
|
||||
Exception raised for unsupported parameter formats.
|
||||
"""
|
||||
|
||||
def __init__(self, param_name):
|
||||
super().__init__(f"Parameters format '{param_name}' not supported.")
|
||||
940
ci/data_extractor/src/formatter.py
Normal file
940
ci/data_extractor/src/formatter.py
Normal file
@@ -0,0 +1,940 @@
|
||||
import collections
|
||||
import copy
|
||||
import enum
|
||||
import pathlib
|
||||
import xml.dom.minidom
|
||||
from collections.abc import Callable
|
||||
|
||||
import svg
|
||||
from benchmark_specs import (ALL_RUST_TYPES, Backend, BenchDetails,
|
||||
CoreCryptoOperation, ErrorFailureProbability,
|
||||
Layer, NoiseDistribution, OperandType, PBSKind,
|
||||
RustType)
|
||||
from py_markdown_table.markdown_table import markdown_table
|
||||
|
||||
|
||||
def compute_comparisons(*results):
|
||||
"""
|
||||
Compute gains for data in ``results``. The first element is considered as the baseline.
|
||||
|
||||
:param results: :class:`list` of results
|
||||
|
||||
:return: :class:`list` of gains with length of ``results`` minus one (baseline is not sent back)
|
||||
"""
|
||||
gains = []
|
||||
baseline = results[0]
|
||||
|
||||
for compared_results in results[1:]:
|
||||
for key, base_value in baseline.items():
|
||||
try:
|
||||
compared_value = compared_results[key]
|
||||
except KeyError:
|
||||
# Ignore missing entries
|
||||
continue
|
||||
|
||||
gain = round((base_value - compared_value) * 100 / base_value, 2)
|
||||
compared_results[key] = gain
|
||||
|
||||
gains.append(compared_results)
|
||||
|
||||
return gains
|
||||
|
||||
|
||||
class OperationDisplayName(enum.StrEnum):
|
||||
Negation = "Negation (-)"
|
||||
AddSub = "Add / Sub (+,-)"
|
||||
Mul = "Mul (x)"
|
||||
EqualNotEqual = "Equal / Not Equal (eq, ne)"
|
||||
Comparisons = "Comparisons (ge, gt, le, lt)"
|
||||
MaxMin = "Max / Min (max, min)"
|
||||
Bitwise = "Bitwise operations (&, \\|, ^)"
|
||||
Div = "Div (/)"
|
||||
Rem = "Rem (%)"
|
||||
DivRem = "Div / Rem (/, %)"
|
||||
Shifts = "Left / Right Shifts (<<, >>)"
|
||||
Rotates = "Left / Right Rotations (left_rotate, right_rotate)"
|
||||
LeadingTrailing = "Leading / Trailing zeros/ones"
|
||||
Log2 = "Log2"
|
||||
Select = "Select"
|
||||
|
||||
|
||||
class BenchArray:
|
||||
def __init__(self, array, layer, metadata: dict = None):
|
||||
self.array = array
|
||||
self.layer = layer
|
||||
self.metadata = metadata
|
||||
|
||||
def __repr__(self):
|
||||
return f"BenchArray(layer={self.layer}, metadata={self.metadata})"
|
||||
|
||||
|
||||
class GenericFormatter:
|
||||
def __init__(
|
||||
self,
|
||||
layer: Layer,
|
||||
backend: Backend,
|
||||
pbs_kind: PBSKind,
|
||||
grouping_factor: int = None,
|
||||
):
|
||||
"""
|
||||
Generic formatter for a given specified layer, backend, and PBS kind.
|
||||
|
||||
:param layer: Represents the layer configuration for the object.
|
||||
:type layer: Layer
|
||||
:param backend: Specifies the backend system to be used.
|
||||
:type backend: Backend
|
||||
:param pbs_kind: Specifies the type or kind of PBS (Private Backend System).
|
||||
:type pbs_kind: PBSKind
|
||||
:param grouping_factor: Specifies the multi-bit PBS grouping factor to use a filter.
|
||||
:type grouping_factor: int, optional
|
||||
"""
|
||||
self.layer = layer
|
||||
self.backend = backend
|
||||
self.pbs_kind = pbs_kind
|
||||
self.requested_grouping_factor = grouping_factor
|
||||
|
||||
def set_grouping_factor(self, grouping_factor: int):
|
||||
"""
|
||||
Sets the grouping factor for a computation or operation.
|
||||
|
||||
This method allows configuration of the requested grouping factor
|
||||
value, which can affect how operations using multi-bit parameters set are processed.
|
||||
|
||||
:param grouping_factor: The desired grouping factor to use.
|
||||
:type grouping_factor: int
|
||||
"""
|
||||
self.requested_grouping_factor = grouping_factor
|
||||
|
||||
def format_data(
|
||||
self, data: dict[BenchDetails : list[int]], conversion_func: Callable
|
||||
):
|
||||
"""
|
||||
Formats data based on the specified layer and applies a conversion function to
|
||||
transform the data.
|
||||
|
||||
The method determines the data formatting logic by matching the
|
||||
current layer of the object and invokes the appropriate specific layer-related
|
||||
formatting function.
|
||||
|
||||
:param data: A dictionary where the keys are instances of `BenchDetails` and
|
||||
the values are lists of integers, representing the benchmark data to be
|
||||
formatted.
|
||||
:type data: dict[BenchDetails : list[int]]
|
||||
:param conversion_func: A callable function that will be applied to transform
|
||||
the data values based on the specific layer requirements.
|
||||
:type conversion_func: Callable
|
||||
|
||||
:return: The formatted data results after applying layer and conversion logic.
|
||||
:rtype: Any
|
||||
|
||||
:raises NotImplementedError: Raised when the specified layer is unsupported.
|
||||
"""
|
||||
match self.layer:
|
||||
case Layer.Integer:
|
||||
return self._format_integer_data(data, conversion_func)
|
||||
case Layer.CoreCrypto:
|
||||
return self._format_core_crypto_data(data, conversion_func)
|
||||
case _:
|
||||
raise NotImplementedError(f"layer '{self.layer}' not supported yet")
|
||||
|
||||
@staticmethod
|
||||
def _format_integer_data(data: dict[BenchDetails : list[int]], conversion_func):
|
||||
formatted = collections.defaultdict(
|
||||
lambda: {
|
||||
2: "N/A",
|
||||
8: "N/A",
|
||||
16: "N/A",
|
||||
32: "N/A",
|
||||
64: "N/A",
|
||||
128: "N/A",
|
||||
256: "N/A",
|
||||
}
|
||||
)
|
||||
for details, timings in data.items():
|
||||
test_name = "_".join((details.sign_flavor.value, details.operation_name))
|
||||
bit_width = details.bit_size
|
||||
value = conversion_func(timings[-1])
|
||||
|
||||
if bit_width == 40:
|
||||
# Ignore this width as it's not displayed publicly.
|
||||
continue
|
||||
|
||||
formatted[test_name][bit_width] = value
|
||||
|
||||
return formatted
|
||||
|
||||
@staticmethod
|
||||
def _format_core_crypto_data(data: dict[BenchDetails : list[int]], conversion_func):
|
||||
params_set = set()
|
||||
for details in data:
|
||||
try:
|
||||
params_set.add(details.get_params_definition())
|
||||
except Exception:
|
||||
# Might be a Boolean parameters set, ignoring
|
||||
continue
|
||||
|
||||
params_set = sorted(params_set)
|
||||
|
||||
formatted = collections.defaultdict(
|
||||
lambda: {params: "N/A" for params in params_set}
|
||||
)
|
||||
for details, timings in data.items():
|
||||
try:
|
||||
reduced_params = details.get_params_definition()
|
||||
except Exception:
|
||||
# Might be a Boolean parameters set, ignoring
|
||||
continue
|
||||
|
||||
test_name = details.operation_name
|
||||
value = conversion_func(timings[-1])
|
||||
formatted[test_name][reduced_params] = value
|
||||
|
||||
return formatted
|
||||
|
||||
def generate_array(
|
||||
self,
|
||||
data,
|
||||
operand_type: OperandType = None,
|
||||
excluded_types: list[RustType] = None,
|
||||
) -> list[BenchArray]:
|
||||
"""
|
||||
Generates an array of `BenchArray` based on the specified layer and criteria.
|
||||
|
||||
This method takes input data and generates an array of `BenchArray` objects,
|
||||
using the rules defined by the current `layer`. The behavior varies depending
|
||||
on the active layer, and certain types can be explicitly excluded from the
|
||||
generation process.
|
||||
|
||||
:param data: Input data to generate the array from.
|
||||
:type data: Any
|
||||
:param operand_type: Specifies the type of operand to guide the array generation.
|
||||
Defaults to `None`.
|
||||
:type operand_type: OperandType, optional
|
||||
:param excluded_types: A list of `RustType` to exclude from array generation.
|
||||
Defaults to `None`.
|
||||
:type excluded_types: list[RustType], optional
|
||||
|
||||
:return: A list of generated `BenchArray` objects.
|
||||
:rtype: list[BenchArray]
|
||||
|
||||
:raises NotImplementedError: If the current layer is not implemented.
|
||||
"""
|
||||
match self.layer:
|
||||
case Layer.Integer:
|
||||
return self._generate_unsigned_integer_array(
|
||||
data, operand_type, excluded_types
|
||||
)
|
||||
case Layer.CoreCrypto:
|
||||
return self._generate_core_crypto_showcase_arrays(data)
|
||||
case _:
|
||||
raise NotImplementedError
|
||||
|
||||
def _generate_unsigned_integer_array(
|
||||
self,
|
||||
data,
|
||||
operand_type: OperandType = None,
|
||||
excluded_types: list[RustType] = None,
|
||||
):
|
||||
match operand_type:
|
||||
case OperandType.CipherText:
|
||||
prefix = "unsigned"
|
||||
case OperandType.PlainText:
|
||||
prefix = "unsigned_scalar"
|
||||
|
||||
match self.backend:
|
||||
case Backend.CPU:
|
||||
operations = [
|
||||
f"{prefix}_neg_parallelized",
|
||||
f"{prefix}_add_parallelized",
|
||||
f"{prefix}_mul_parallelized",
|
||||
f"{prefix}_eq_parallelized",
|
||||
f"{prefix}_gt_parallelized",
|
||||
f"{prefix}_max_parallelized",
|
||||
f"{prefix}_bitand_parallelized",
|
||||
f"{prefix}_div_rem_parallelized",
|
||||
f"{prefix}_left_shift_parallelized",
|
||||
f"{prefix}_rotate_left_parallelized",
|
||||
f"{prefix}_leading_zeros_parallelized",
|
||||
f"{prefix}_ilog2_parallelized",
|
||||
f"{prefix}_if_then_else_parallelized",
|
||||
]
|
||||
case Backend.GPU:
|
||||
match operand_type:
|
||||
case OperandType.CipherText:
|
||||
prefix = "cuda"
|
||||
case OperandType.PlainText:
|
||||
prefix = "cuda_scalar"
|
||||
|
||||
operations = [
|
||||
f"{prefix}_neg",
|
||||
f"{prefix}_add",
|
||||
f"{prefix}_mul",
|
||||
f"{prefix}_eq",
|
||||
f"{prefix}_gt",
|
||||
f"{prefix}_max",
|
||||
f"{prefix}_bitand",
|
||||
f"{prefix}_div_rem",
|
||||
f"{prefix}_left_shift",
|
||||
f"{prefix}_rotate_left",
|
||||
f"{prefix}_leading_zeros",
|
||||
f"{prefix}_ilog2",
|
||||
f"{prefix}_if_then_else",
|
||||
]
|
||||
case _:
|
||||
raise NotImplementedError(
|
||||
f"backend '{self.backend}' not supported yet for integer formatting"
|
||||
)
|
||||
|
||||
display_names = [
|
||||
OperationDisplayName.Negation,
|
||||
OperationDisplayName.AddSub,
|
||||
OperationDisplayName.Mul,
|
||||
OperationDisplayName.EqualNotEqual,
|
||||
OperationDisplayName.Comparisons,
|
||||
OperationDisplayName.MaxMin,
|
||||
OperationDisplayName.Bitwise,
|
||||
OperationDisplayName.DivRem,
|
||||
OperationDisplayName.Shifts,
|
||||
OperationDisplayName.Rotates,
|
||||
OperationDisplayName.LeadingTrailing,
|
||||
OperationDisplayName.Log2,
|
||||
OperationDisplayName.Select,
|
||||
]
|
||||
|
||||
types = ALL_RUST_TYPES.copy()
|
||||
excluded_types = excluded_types if excluded_types is not None else []
|
||||
for excluded in excluded_types:
|
||||
types.remove(excluded)
|
||||
|
||||
first_column_header = "Operation \\ Size"
|
||||
|
||||
# Adapt list to plaintext benchmarks results.
|
||||
if operand_type == OperandType.PlainText:
|
||||
if self.backend == Backend.CPU:
|
||||
div_name = f"{prefix}_div_parallelized"
|
||||
rem_name = f"{prefix}_rem_parallelized"
|
||||
elif self.backend == Backend.GPU:
|
||||
div_name = f"{prefix}_div"
|
||||
rem_name = f"{prefix}_rem"
|
||||
|
||||
operations.insert(8, div_name)
|
||||
operations.insert(9, rem_name)
|
||||
operations.pop(7) # Remove div_rem_parallelized
|
||||
|
||||
display_names.insert(
|
||||
8,
|
||||
OperationDisplayName.Div,
|
||||
)
|
||||
display_names.insert(
|
||||
9,
|
||||
OperationDisplayName.Rem,
|
||||
)
|
||||
display_names.pop(7) # Remove Div / Rem
|
||||
|
||||
# Negation operation doesn't exist in plaintext
|
||||
operations.pop(0)
|
||||
display_names.pop(0)
|
||||
|
||||
data_without_excluded_types = copy.deepcopy(data)
|
||||
for v in data_without_excluded_types.values():
|
||||
for excluded in excluded_types:
|
||||
try:
|
||||
v.pop(excluded.value)
|
||||
except KeyError:
|
||||
# Type is not contained in the results, ignoring
|
||||
continue
|
||||
|
||||
filtered_data = filter(lambda t: t in operations, data_without_excluded_types)
|
||||
# Get operation names as key of the dict to ease fetching
|
||||
filtered_data_dict = {
|
||||
item: tuple(data_without_excluded_types[item].values())
|
||||
for item in filtered_data
|
||||
}
|
||||
|
||||
result_lines = []
|
||||
for name, op in zip(display_names, operations):
|
||||
try:
|
||||
line = {first_column_header: name.value}
|
||||
line.update(
|
||||
{
|
||||
types[i].name: value
|
||||
for i, value in enumerate(filtered_data_dict[op])
|
||||
}
|
||||
)
|
||||
result_lines.append(line)
|
||||
except KeyError:
|
||||
# Operation not found in the results, ignoring this line.
|
||||
print(
|
||||
f"backend '{self.backend}' could not find operation '{op}' to put in line '{name}'"
|
||||
)
|
||||
continue
|
||||
|
||||
return [
|
||||
BenchArray(result_lines, self.layer),
|
||||
]
|
||||
|
||||
def _build_results_dict(
|
||||
self,
|
||||
pfails: list[ErrorFailureProbability],
|
||||
noise_distributions: list[NoiseDistribution],
|
||||
operation_displays: list[CoreCryptoOperation],
|
||||
default_precisions: Callable[[], dict],
|
||||
):
|
||||
results_dict = {}
|
||||
|
||||
for pfail in pfails:
|
||||
for noise in noise_distributions:
|
||||
results_dict[CoreCryptoResultsKey(pfail, noise)] = {
|
||||
o: default_precisions() for o in operation_displays
|
||||
}
|
||||
|
||||
return results_dict
|
||||
|
||||
def _generate_core_crypto_showcase_arrays(
|
||||
self,
|
||||
data,
|
||||
):
|
||||
supported_pfails = [
|
||||
ErrorFailureProbability.TWO_MINUS_40,
|
||||
ErrorFailureProbability.TWO_MINUS_64,
|
||||
ErrorFailureProbability.TWO_MINUS_128,
|
||||
]
|
||||
noise_distributions = [
|
||||
NoiseDistribution.Gaussian,
|
||||
NoiseDistribution.TUniform,
|
||||
]
|
||||
|
||||
operation_displays = [op.value for op in OPERATIONS_DISPLAYS]
|
||||
|
||||
sorted_results = self._build_results_dict(
|
||||
supported_pfails,
|
||||
noise_distributions,
|
||||
OPERATIONS_DISPLAYS,
|
||||
DEFAULT_CORE_CRYPTO_PRECISIONS,
|
||||
)
|
||||
|
||||
for operation, timings in data.items():
|
||||
try:
|
||||
formatted_name = CoreCryptoOperation.from_str(operation)
|
||||
except NotImplementedError:
|
||||
# Operation is not supported.
|
||||
continue
|
||||
|
||||
for param_definition, value in timings.items():
|
||||
pfail = param_definition.p_fail
|
||||
if pfail not in supported_pfails:
|
||||
print(f"[{operation}] P-fail '{pfail}' is not supported")
|
||||
continue
|
||||
noise = param_definition.noise_distribution
|
||||
precision = int(param_definition.message_size) * 2
|
||||
key = CoreCryptoResultsKey(pfail, noise)
|
||||
|
||||
if (
|
||||
formatted_name == CoreCryptoOperation.MultiBitPBS
|
||||
or formatted_name == CoreCryptoOperation.KeySwitchMultiBitPBS
|
||||
) and param_definition.pbs_kind != PBSKind.MultiBit:
|
||||
# Skip this operation since a multi-bit operation cannot be done with any other parameters type.
|
||||
continue
|
||||
|
||||
grouping_factor = param_definition.grouping_factor
|
||||
if (
|
||||
grouping_factor is not None
|
||||
and grouping_factor != self.requested_grouping_factor
|
||||
):
|
||||
continue
|
||||
|
||||
if (
|
||||
param_definition.details["variation"]
|
||||
or param_definition.details["trailing_details"]
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
sorted_results[key][formatted_name][precision] = value
|
||||
except KeyError:
|
||||
# Operation is not supposed to appear in the formatted array.
|
||||
continue
|
||||
|
||||
first_column_header = "Operation \\ Precision (bits)"
|
||||
|
||||
arrays = []
|
||||
for key, results in sorted_results.items():
|
||||
array = []
|
||||
for operation, timings in results.items():
|
||||
d = {first_column_header: operation.value}
|
||||
d.update({str(k): v for k, v in timings.items()})
|
||||
array.append(d)
|
||||
|
||||
arrays.append(
|
||||
BenchArray(
|
||||
array,
|
||||
self.layer,
|
||||
metadata={"pfail": key.pfail, "noise": key.noise_distribution},
|
||||
)
|
||||
)
|
||||
|
||||
return arrays
|
||||
|
||||
|
||||
# ---------------------------
|
||||
# Core_crypto layer constants
|
||||
# ---------------------------
|
||||
|
||||
OPERATIONS_DISPLAYS = [
|
||||
# CoreCryptoOperation.KeySwitch, # Uncomment this line to get keyswitch in the tables
|
||||
CoreCryptoOperation.PBS,
|
||||
CoreCryptoOperation.MultiBitPBS,
|
||||
CoreCryptoOperation.KeyswitchPBS,
|
||||
CoreCryptoOperation.KeySwitchMultiBitPBS,
|
||||
]
|
||||
|
||||
DEFAULT_CORE_CRYPTO_PRECISIONS = lambda: {
|
||||
2: "N/A",
|
||||
4: "N/A",
|
||||
6: "N/A",
|
||||
8: "N/A",
|
||||
}
|
||||
|
||||
|
||||
class CoreCryptoResultsKey:
|
||||
"""
|
||||
Representation of a hashable result key for the core_crypto layer.
|
||||
|
||||
:param pfail: Probability of failure associated with the cryptographic result.
|
||||
:type pfail: ErrorFailureProbability
|
||||
:param noise_distribution: Noise distribution parameter linked to the
|
||||
cryptographic result.
|
||||
:type noise_distribution: NoiseDistribution
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, pfail: ErrorFailureProbability, noise_distribution: NoiseDistribution
|
||||
):
|
||||
self.pfail = pfail
|
||||
self.noise_distribution = noise_distribution
|
||||
|
||||
def __eq__(self, other):
|
||||
return (
|
||||
self.pfail == other.pfail
|
||||
and self.noise_distribution == other.noise_distribution
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.pfail, self.noise_distribution))
|
||||
|
||||
def __repr__(self):
|
||||
return f"CoreCryptoResultsKey(pfail={self.pfail}, noise_distribution={self.noise_distribution})"
|
||||
|
||||
|
||||
class CSVFormatter(GenericFormatter):
|
||||
"""
|
||||
Formatter to generate CSV content.
|
||||
"""
|
||||
|
||||
def generate_csv(self, data: dict[str, collections.defaultdict]) -> list[list]:
|
||||
"""
|
||||
Generates a CSV-compatible data structure based on the provided input data and
|
||||
the current layer type. The method processes the input to construct headers and
|
||||
rows suitable for CSV writing.
|
||||
|
||||
:param data: A dictionary where keys represent row identifiers and values
|
||||
are dictionaries of column-value pairs representing the data for
|
||||
each row.
|
||||
:type data: dict[str, collections.defaultdict]
|
||||
|
||||
:return: A list of lists where each sub-list represents a row in the CSV,
|
||||
including the header row.
|
||||
:rtype: list
|
||||
|
||||
:raises NotImplementedError: If the layer type specified in the object's
|
||||
`layer` attribute is unsupported.
|
||||
"""
|
||||
headers_values = data.get(list(data)[0]).keys()
|
||||
|
||||
match self.layer:
|
||||
case Layer.Integer:
|
||||
headers = ["Operation \\ Size(bit)", *headers_values]
|
||||
case Layer.CoreCrypto:
|
||||
headers = ["Operation \\ Parameters set", *headers_values]
|
||||
case _:
|
||||
print(
|
||||
f"tfhe-rs layer '{self.layer}' currently not supported for CSV writing"
|
||||
)
|
||||
raise NotImplementedError
|
||||
|
||||
csv_data = [headers]
|
||||
csv_data.extend(
|
||||
[[key, *list(values_dict.values())] for key, values_dict in data.items()]
|
||||
)
|
||||
|
||||
return csv_data
|
||||
|
||||
|
||||
class MarkdownFormatter(GenericFormatter):
|
||||
"""
|
||||
Formatter to generate Markdown content.
|
||||
"""
|
||||
|
||||
def generate_markdown_array(
|
||||
self,
|
||||
generic_array: BenchArray,
|
||||
) -> str:
|
||||
"""
|
||||
Generates a Markdown representation of the provided generic array.
|
||||
|
||||
:param generic_array: The input array encapsulated in a BenchArray object.
|
||||
|
||||
:return: A Markdown formatted string representing the input array.
|
||||
:rtype: str
|
||||
"""
|
||||
md_array = (
|
||||
markdown_table(generic_array.array)
|
||||
.set_params(row_sep="markdown", quote=False, padding_weight="right")
|
||||
.get_markdown()
|
||||
)
|
||||
|
||||
return md_array
|
||||
|
||||
|
||||
# -------------
|
||||
# SVG constants
|
||||
# -------------
|
||||
|
||||
BLACK_COLOR = "black"
|
||||
WHITE_COLOR = "white"
|
||||
LIGHT_GREY_COLOR = "#f3f3f3"
|
||||
YELLOW_COLOR = "#fbbc04"
|
||||
|
||||
FONT_FAMILY = "Arial"
|
||||
FONT_SIZE = 14
|
||||
|
||||
BORDER_WIDTH_PIXEL = 2
|
||||
# Operation name is always in table first column
|
||||
OPERATION_NAME_HORIZONTAL_POSITION = 6
|
||||
|
||||
SPECIAL_CHARS_PAIRS = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
"\\|": "|",
|
||||
}
|
||||
|
||||
# -------------
|
||||
|
||||
|
||||
class SVGFormatter(GenericFormatter):
|
||||
"""
|
||||
Formatter to generate SVG content.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _transform_special_characters(strg: str):
|
||||
for char, replacement in SPECIAL_CHARS_PAIRS.items():
|
||||
if char in strg:
|
||||
strg = strg.replace(char, replacement)
|
||||
|
||||
return strg
|
||||
|
||||
def _build_svg_headers_row(
|
||||
self,
|
||||
layer: Layer,
|
||||
headers,
|
||||
overall_width,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
):
|
||||
op_header = headers.pop(0)
|
||||
header_elements = [
|
||||
svg.Rect(
|
||||
x=0, y=0, width=overall_width, height=row_height, fill=BLACK_COLOR
|
||||
),
|
||||
self._build_svg_text(
|
||||
OPERATION_NAME_HORIZONTAL_POSITION,
|
||||
row_height / 2,
|
||||
op_header,
|
||||
text_anchor="start",
|
||||
fill=WHITE_COLOR,
|
||||
font_weight="bold",
|
||||
),
|
||||
]
|
||||
|
||||
for row_idx, type_ident in enumerate(headers):
|
||||
curr_x = op_name_col_width + row_idx * per_timing_col_width
|
||||
|
||||
match layer:
|
||||
case Layer.Integer:
|
||||
type_name_width = type_ident.strip("FheUint")
|
||||
header_elements.extend(
|
||||
[
|
||||
# Rust type class
|
||||
self._build_svg_text(
|
||||
curr_x + per_timing_col_width / 2,
|
||||
row_height / 3,
|
||||
"FheUint",
|
||||
fill=WHITE_COLOR,
|
||||
font_weight="bold",
|
||||
),
|
||||
# Actual size of the Rust type
|
||||
self._build_svg_text(
|
||||
curr_x + per_timing_col_width / 2,
|
||||
2 * row_height / 3 + 3,
|
||||
type_name_width,
|
||||
fill=WHITE_COLOR,
|
||||
font_weight="bold",
|
||||
),
|
||||
]
|
||||
)
|
||||
case Layer.CoreCrypto:
|
||||
header_elements.append(
|
||||
# Core_crypto arrays contains only ciphertext modulus size as headers
|
||||
self._build_svg_text(
|
||||
curr_x + per_timing_col_width / 2,
|
||||
row_height / 2,
|
||||
type_ident,
|
||||
fill=WHITE_COLOR,
|
||||
font_weight="bold",
|
||||
)
|
||||
)
|
||||
case _:
|
||||
raise NotImplementedError
|
||||
|
||||
return header_elements
|
||||
|
||||
def _build_svg_timing_row(
|
||||
self,
|
||||
timings_row,
|
||||
row_y_pos,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
):
|
||||
timing_elements = []
|
||||
op_name = timings_row.pop(0)
|
||||
timing_elements.append(
|
||||
self._build_svg_text(
|
||||
OPERATION_NAME_HORIZONTAL_POSITION,
|
||||
row_y_pos + row_height / 2,
|
||||
op_name,
|
||||
text_anchor="start",
|
||||
)
|
||||
)
|
||||
for timing_idx, timing in enumerate(timings_row):
|
||||
timing_elements.append(
|
||||
self._build_svg_text(
|
||||
op_name_col_width
|
||||
+ timing_idx * per_timing_col_width
|
||||
+ per_timing_col_width / 2,
|
||||
row_y_pos + row_height / 2,
|
||||
timing,
|
||||
)
|
||||
)
|
||||
|
||||
return timing_elements
|
||||
|
||||
def _build_svg_text(
|
||||
self, x, y, text, text_anchor="middle", fill=BLACK_COLOR, font_weight="normal"
|
||||
):
|
||||
return svg.Text(
|
||||
x=x,
|
||||
y=y,
|
||||
dominant_baseline="middle",
|
||||
text_anchor=text_anchor,
|
||||
font_family=FONT_FAMILY,
|
||||
font_size=FONT_SIZE,
|
||||
fill=fill,
|
||||
text=text,
|
||||
font_weight=font_weight,
|
||||
)
|
||||
|
||||
def _build_svg_borders(
|
||||
self,
|
||||
overall_width,
|
||||
overall_height,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
row_count,
|
||||
col_count,
|
||||
):
|
||||
border_elements = []
|
||||
|
||||
# Horizontal borders, scrolling vertically
|
||||
for row_idx in range(row_count + 2):
|
||||
row_y = row_idx * row_height
|
||||
border_elements.append(
|
||||
svg.Line(
|
||||
x1=0,
|
||||
y1=row_y,
|
||||
x2=overall_width,
|
||||
y2=row_y,
|
||||
stroke=WHITE_COLOR,
|
||||
stroke_width=BORDER_WIDTH_PIXEL,
|
||||
)
|
||||
)
|
||||
|
||||
# Vertical borders, scrolling horizontally
|
||||
# Left border
|
||||
border_elements.append(
|
||||
svg.Line(
|
||||
x1=0,
|
||||
y1=0,
|
||||
x2=0,
|
||||
y2=overall_height,
|
||||
stroke=WHITE_COLOR,
|
||||
stroke_width=BORDER_WIDTH_PIXEL,
|
||||
)
|
||||
)
|
||||
# Timing cols
|
||||
for col_idx in range(col_count + 1):
|
||||
col_x = op_name_col_width + col_idx * per_timing_col_width
|
||||
border_elements.append(
|
||||
svg.Line(
|
||||
x1=col_x,
|
||||
y1=0,
|
||||
x2=col_x,
|
||||
y2=overall_height,
|
||||
stroke=WHITE_COLOR,
|
||||
stroke_width=BORDER_WIDTH_PIXEL,
|
||||
)
|
||||
)
|
||||
|
||||
return border_elements
|
||||
|
||||
def generate_svg_table(self, generic_array: BenchArray) -> str:
|
||||
"""
|
||||
Generates an SVG representation of a table from the given `BenchArray` object.
|
||||
This method processes array data to create a visual representation of the
|
||||
provided headers and values as an SVG image, organizing them into rows and
|
||||
columns consistent with the structure of the input.
|
||||
|
||||
:param generic_array: The BenchArray object containing structured data for
|
||||
generating the table.
|
||||
:type generic_array: BenchArray
|
||||
|
||||
:return: An SVG representation of the table as a well-formatted string.
|
||||
The SVG is generated with appropriate dimensions, colors, and styles
|
||||
to ensure a visually clear and consistent layout.
|
||||
:rtype: str
|
||||
"""
|
||||
headers = list(generic_array.array[0].keys())
|
||||
ops = generic_array.array[:]
|
||||
|
||||
# TODO Create a class to handle table dimension which will depend on tfhe-rs layer
|
||||
col_count = len(headers) - 1
|
||||
row_count = 1 + len(ops)
|
||||
|
||||
overall_width = int(round(900 / 1.25))
|
||||
row_height = int(round(50 / 1.25))
|
||||
op_name_col_width = int(round(375 / 1.25))
|
||||
per_timing_col_width = (overall_width - op_name_col_width) / col_count
|
||||
overall_height = row_count * row_height
|
||||
|
||||
svg_elements = []
|
||||
|
||||
# Generate headers row
|
||||
svg_elements.extend(
|
||||
self._build_svg_headers_row(
|
||||
self.layer,
|
||||
headers,
|
||||
overall_width,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
)
|
||||
)
|
||||
|
||||
# Generate operations rectangle
|
||||
yellow_rect_for_op_names = svg.Rect(
|
||||
x=0,
|
||||
y=row_height,
|
||||
width=op_name_col_width,
|
||||
height=overall_height - row_height,
|
||||
fill=YELLOW_COLOR,
|
||||
)
|
||||
svg_elements.append(yellow_rect_for_op_names)
|
||||
|
||||
# Generate timings rectangle
|
||||
grey_rect_for_timings = svg.Rect(
|
||||
x=op_name_col_width,
|
||||
y=row_height,
|
||||
width=overall_width - op_name_col_width,
|
||||
height=overall_height - row_height,
|
||||
fill=LIGHT_GREY_COLOR,
|
||||
)
|
||||
svg_elements.append(grey_rect_for_timings)
|
||||
|
||||
for row_count, row in enumerate(ops):
|
||||
row_y = row_height + row_count * row_height
|
||||
row_split = [
|
||||
self._transform_special_characters(v)
|
||||
for v in filter(None, row.values())
|
||||
]
|
||||
svg_elements.extend(
|
||||
self._build_svg_timing_row(
|
||||
row_split,
|
||||
row_y,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
)
|
||||
)
|
||||
|
||||
# Generate borders
|
||||
svg_elements.extend(
|
||||
self._build_svg_borders(
|
||||
overall_width,
|
||||
overall_height,
|
||||
row_height,
|
||||
op_name_col_width,
|
||||
per_timing_col_width,
|
||||
row_count,
|
||||
col_count,
|
||||
)
|
||||
)
|
||||
|
||||
canvas = svg.SVG(
|
||||
width="100%",
|
||||
height=overall_height,
|
||||
viewBox=f"0 0 {overall_width} {overall_height}",
|
||||
elements=svg_elements,
|
||||
preserveAspectRatio="meet",
|
||||
)
|
||||
|
||||
dom = xml.dom.minidom.parseString(str(canvas))
|
||||
return dom.toprettyxml()
|
||||
|
||||
def generate_svg_table_from_markdown_file(
|
||||
self,
|
||||
input_filepath: str,
|
||||
) -> str:
|
||||
"""
|
||||
Generates an SVG table from a Markdown file.
|
||||
|
||||
This method reads a Markdown file, parses the table content, and transforms
|
||||
it into an SVG representation. The input Markdown file must have headers
|
||||
defined in the first row and rows of data separated by the "|" character.
|
||||
|
||||
:param input_filepath: The file path to the Markdown file.
|
||||
:type input_filepath: str
|
||||
|
||||
:return: The generated SVG representation of the parsed Markdown table.
|
||||
:rtype: str
|
||||
"""
|
||||
with pathlib.Path(input_filepath).open("r") as f:
|
||||
md_lines = f.read().splitlines()
|
||||
|
||||
headers = [h.lstrip().rstrip() for h in md_lines.pop(0).split("|")[1:-1]]
|
||||
|
||||
md_lines.pop(0) # remove separation line
|
||||
parsed_data = []
|
||||
for line in md_lines:
|
||||
values = [v.lstrip().rstrip() for v in line.split("|")[1:-1]]
|
||||
if len(values) > len(headers):
|
||||
# A '|' character is contained in the operation name
|
||||
values[0] += "".join(["|", values.pop(1)])
|
||||
parsed_data.append({k: v for k, v in zip(headers, values)})
|
||||
|
||||
array = BenchArray(parsed_data, self.layer)
|
||||
|
||||
return self.generate_svg_table(array)
|
||||
183
ci/data_extractor/src/regression.py
Normal file
183
ci/data_extractor/src/regression.py
Normal file
@@ -0,0 +1,183 @@
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import config
|
||||
import connector
|
||||
from benchmark_specs import Backend, Layer
|
||||
from exceptions import NoDataFound
|
||||
|
||||
import utils
|
||||
|
||||
try:
|
||||
import tomllib # Python v3.11+
|
||||
except ModuleNotFoundError:
|
||||
import pip._vendor.tomli as tomllib # the same tomllib that's now included in Python v3.11+
|
||||
|
||||
|
||||
def generate_json_regression_file(
|
||||
conn: connector.PostgreConnector, ops_filter: dict, user_config: config.UserConfig
|
||||
):
|
||||
"""
|
||||
Generate a JSON regression file based on benchmark data from specified branches
|
||||
and operations.
|
||||
|
||||
This function extracts benchmark data for a set of operations across HEAD
|
||||
and BASE branches, processes it, and generates a structured JSON output
|
||||
representing the benchmarking comparison.
|
||||
|
||||
:param conn: Connector to the benchmark data source.
|
||||
:type conn: Any
|
||||
:param ops_filter: A dictionary containing operations as keys and their
|
||||
respective filters as values.
|
||||
:type ops_filter: dict
|
||||
:param user_config: A configuration object that contains branch information,
|
||||
backend settings, and output preferences for regression.
|
||||
:type user_config: UserConfig
|
||||
|
||||
:raises NoDataFound: If benchmark data is not found for the HEAD or BASE branch
|
||||
during processing.
|
||||
:raises json.JSONDecodeError: If the final dictionary cannot be converted into
|
||||
JSON format.
|
||||
"""
|
||||
error_msg = "Cannot generate JSON regression file (error: no data found on {} branch '{}' (layer: {}, operations: {}))"
|
||||
regression_data = {}
|
||||
|
||||
for bench_target, ops in ops_filter.items():
|
||||
layer_name, _, bench_id = bench_target.partition("-")
|
||||
layer = Layer.from_str(layer_name)
|
||||
|
||||
try:
|
||||
head_branch_data = conn.fetch_benchmark_data(
|
||||
user_config,
|
||||
operation_filter=ops,
|
||||
layer=layer,
|
||||
branch=user_config.head_branch,
|
||||
name_suffix="_mean_regression",
|
||||
last_value_only=True,
|
||||
)
|
||||
except NoDataFound:
|
||||
print(error_msg.format("HEAD", user_config.head_branch, layer, ops))
|
||||
raise
|
||||
|
||||
try:
|
||||
base_branch_data = conn.fetch_benchmark_data(
|
||||
user_config,
|
||||
operation_filter=ops,
|
||||
layer=layer,
|
||||
branch=user_config.base_branch,
|
||||
last_value_only=False,
|
||||
)
|
||||
except NoDataFound:
|
||||
print(error_msg.format("BASE", user_config.base_branch, layer, ops))
|
||||
raise
|
||||
|
||||
for bench_details, values in head_branch_data.items():
|
||||
regression_data[bench_details.operation_name] = {
|
||||
"name": bench_details.operation_name,
|
||||
"bit_size": bench_details.bit_size,
|
||||
"params": bench_details.params,
|
||||
"results": {
|
||||
"head": {"name": user_config.head_branch, "value": values[0]},
|
||||
},
|
||||
}
|
||||
|
||||
for bench_details, values in base_branch_data.items():
|
||||
try:
|
||||
reg = regression_data[bench_details.operation_name]
|
||||
if (
|
||||
reg["bit_size"] == bench_details.bit_size
|
||||
and reg["params"] == bench_details.params
|
||||
):
|
||||
reg["results"]["base"] = {
|
||||
"name": user_config.base_branch,
|
||||
"data": values,
|
||||
}
|
||||
except KeyError:
|
||||
# No value exists on the head branch for this key, ignoring the base branch result.
|
||||
continue
|
||||
|
||||
final_dict = {
|
||||
"backend": user_config.backend,
|
||||
"profile": user_config.regression_selected_profile,
|
||||
"operations": list(regression_data.values()),
|
||||
}
|
||||
|
||||
output_file = "".join([user_config.output_file, ".json"])
|
||||
utils.write_to_json(final_dict, output_file)
|
||||
|
||||
|
||||
def parse_toml_file(
|
||||
path: str, backend: Backend, profile_name: str
|
||||
) -> dict[str, list[str]]:
|
||||
"""
|
||||
Parse a TOML file defining regression profiles and return its content as a dictionary.
|
||||
|
||||
:param path: path to TOML file
|
||||
:type path: str
|
||||
:param backend: type of backend used for the benchmarks
|
||||
:type backend: Backend
|
||||
:param profile_name: name of the regression profile
|
||||
:type profile_name: str
|
||||
|
||||
:return: profile content formatted as `{"<benchmark_target>": ["<operation_1>", ...]}`
|
||||
:rtype: dict[str, list[str]]
|
||||
"""
|
||||
file_path = pathlib.Path(path)
|
||||
try:
|
||||
return tomllib.loads(file_path.read_text())[backend][profile_name]
|
||||
except tomllib.TOMLDecodeError as err:
|
||||
raise RuntimeError(f"failed to parse definition file (error: {err})")
|
||||
except KeyError:
|
||||
raise RuntimeError(
|
||||
f"failed to find definition profile (error: profile '{profile_name}' cannot be found in '{file_path}')"
|
||||
)
|
||||
|
||||
|
||||
def perform_regression_json_generation(
|
||||
conn: connector.PostgreConnector, user_config: config.UserConfig
|
||||
):
|
||||
"""
|
||||
Generates a JSON file for regression testing based on the selected regression profile.
|
||||
|
||||
This function generates a JSON file with benchmarks for the specified regression profile.
|
||||
The function requires the `regression_selected_profile` attribute of the `user_config` parameter if
|
||||
custom regression profile operations are to be filtered. If no such profile is provided or no profile
|
||||
definitions file is specified, the function terminates the program with an appropriate message.
|
||||
|
||||
:param conn: PostgreConnector object used for generating the JSON regression file.
|
||||
:type conn: Any
|
||||
:param user_config: Configuration object containing user-defined regression profiles and settings.
|
||||
:type user_config: UserConfig
|
||||
|
||||
:raises SystemExit: If no regression profile is selected, or if the regression profiles
|
||||
file is not provided
|
||||
"""
|
||||
# TODO Currently this implementation doesn't support custom regression profile.
|
||||
# It will return strictly the benchmarks results for the operations defined for the selected profile.
|
||||
# One way to handle that case would be to pass the custom profile as program input (could be either via a string or path containing the profile).
|
||||
if not user_config.regression_selected_profile:
|
||||
print(
|
||||
"Regression generation requires a profile name (see: --regression-selected-profile input argument)"
|
||||
)
|
||||
sys.exit(5)
|
||||
selected_profile = user_config.regression_selected_profile
|
||||
|
||||
operations_filter = {}
|
||||
if user_config.regression_profiles_path:
|
||||
profile_definition = parse_toml_file(
|
||||
user_config.regression_profiles_path, user_config.backend, selected_profile
|
||||
)
|
||||
operations_filter = profile_definition["target"]
|
||||
try:
|
||||
user_config.pbs_kind = profile_definition["env"]["bench_param_type"].lower()
|
||||
except KeyError:
|
||||
# Benchmark parameters type is not declared in the regression definition file
|
||||
# Ignoring
|
||||
pass
|
||||
else:
|
||||
print(
|
||||
"Regression generation requires a profile definitions file (see: --regression-profiles input argument)"
|
||||
)
|
||||
sys.exit(5)
|
||||
|
||||
generate_json_regression_file(conn, operations_filter, user_config)
|
||||
146
ci/data_extractor/src/utils.py
Normal file
146
ci/data_extractor/src/utils.py
Normal file
@@ -0,0 +1,146 @@
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from benchmark_specs import Layer
|
||||
|
||||
SECONDS_IN_NANO = 1e9
|
||||
MILLISECONDS_IN_NANO = 1e6
|
||||
MICROSECONDS_IN_NANO = 1e3
|
||||
|
||||
|
||||
def convert_value_to_readable_text(value: int, max_digits: int = 3) -> str:
|
||||
"""
|
||||
Convert timing in nanoseconds to the highest unit usable.
|
||||
|
||||
:param value: timing value
|
||||
:type value: int
|
||||
:param max_digits: number of digits to keep in the final representation of the value
|
||||
:type max_digits: int, optional
|
||||
|
||||
:return: human-readable value with unit
|
||||
:rtype: str
|
||||
"""
|
||||
if value > SECONDS_IN_NANO:
|
||||
converted_parts = (value / SECONDS_IN_NANO), "s"
|
||||
elif value > MILLISECONDS_IN_NANO:
|
||||
converted_parts = (value / MILLISECONDS_IN_NANO), "ms"
|
||||
elif value > MICROSECONDS_IN_NANO:
|
||||
converted_parts = (value / MICROSECONDS_IN_NANO), "us"
|
||||
else:
|
||||
converted_parts = value, "ns"
|
||||
|
||||
power_of_10 = math.floor(math.log10(converted_parts[0]))
|
||||
rounding_digit = max_digits - (power_of_10 + 1)
|
||||
if converted_parts[0] >= 100.0:
|
||||
rounding_digit = None
|
||||
|
||||
return f"{round(converted_parts[0], rounding_digit)} {converted_parts[1]}"
|
||||
|
||||
|
||||
def convert_gain_to_text(value: float) -> str:
|
||||
"""
|
||||
Convert gains as :class:`float` to :class:`str`
|
||||
|
||||
:param value: gain value
|
||||
:type value: float
|
||||
|
||||
:return: gain as text with percentage sign
|
||||
:rtype: str
|
||||
"""
|
||||
return f"{value} %" if value < 0 else f"+{value} %"
|
||||
|
||||
|
||||
def write_to_csv(lines: list, output_filename: str):
|
||||
"""
|
||||
Write data to a CSV file.
|
||||
|
||||
:param lines: formatted data as iterable
|
||||
:type lines: list
|
||||
:param output_filename: filename where data would be written
|
||||
:type output_filename: str
|
||||
"""
|
||||
with pathlib.Path(output_filename).open("w") as csv_file:
|
||||
writer = csv.writer(csv_file, delimiter=",")
|
||||
for line in lines:
|
||||
writer.writerow(line)
|
||||
|
||||
print(f"Results written as CSV in '{output_filename}'")
|
||||
|
||||
|
||||
def write_to_markdown(lines: str | list[str], output_filename: str):
|
||||
"""
|
||||
Write data to a Markdown file.
|
||||
|
||||
:param lines: formatted lines
|
||||
:type lines: str | list[str]
|
||||
:param output_filename: filename where data would be written
|
||||
:type output_filename: str
|
||||
"""
|
||||
if type(lines) != str:
|
||||
content = "\n".join(lines) + "\n"
|
||||
pathlib.Path(output_filename).write_text(content)
|
||||
else:
|
||||
pathlib.Path(output_filename).write_text(lines)
|
||||
|
||||
print(f"Results written as Markdown in '{output_filename}'")
|
||||
|
||||
|
||||
def write_to_svg(xml_string: str, output_filename: str):
|
||||
"""
|
||||
Write XML to a SVG file.
|
||||
|
||||
:param xml_string: XML formatted string
|
||||
:type xml_string: str
|
||||
:param output_filename: filename where data would be written
|
||||
:type output_filename: str
|
||||
"""
|
||||
pathlib.Path(output_filename).write_text(xml_string)
|
||||
|
||||
print(f"Results written as SVG in '{output_filename}'")
|
||||
|
||||
|
||||
def write_to_json(data: dict, output_filename: str):
|
||||
"""
|
||||
Write data to a JSON file.
|
||||
|
||||
:param data: data that would be dumped as JSON
|
||||
:type data: dict
|
||||
:param output_filename: filename where data would be written
|
||||
:type output_filename: str
|
||||
"""
|
||||
try:
|
||||
dump = json.dumps(data)
|
||||
except json.JSONDecodeError as err:
|
||||
print(f"couldn't convert results into json format (error: {err})")
|
||||
raise
|
||||
|
||||
pathlib.Path(output_filename).write_text(dump)
|
||||
print(f"Results written as JSON in '{output_filename}'")
|
||||
|
||||
|
||||
def append_suffix_to_filename(filename: str, suffix: str, stem: str) -> str:
|
||||
"""
|
||||
Appends a suffix to a given filename, considering a specific stem. If the filename
|
||||
already ends with the given stem, the suffix is inserted before the stem. Otherwise,
|
||||
the suffix and stem are added to the end of the filename.
|
||||
|
||||
:param filename: The original filename to which the suffix and stem will be appended
|
||||
:type filename: str
|
||||
:param suffix: The string to be appended as the suffix
|
||||
:type suffix: str
|
||||
:param stem: The specific stem to check or append after the suffix
|
||||
:type stem: str
|
||||
|
||||
:return: The new filename with the appended suffix and stem
|
||||
:rtype: str
|
||||
"""
|
||||
filename = filename[:] # Make a copy to avoid modifying the original filename
|
||||
if filename.endswith(stem):
|
||||
filename = filename[: -len(stem)] + suffix + stem
|
||||
else:
|
||||
filename += suffix + stem
|
||||
|
||||
return filename
|
||||
@@ -13,13 +13,20 @@ One can also provide a fully custom profile via the issue comment string see: fu
|
||||
|
||||
This script is also capable of checking for performance regression based on previous benchmarks results.
|
||||
It works by providing a result file containing the baseline values and the results of the last run.
|
||||
Alongside this mode, a performance report can be generated to help identify potential regressions.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import enum
|
||||
import json
|
||||
import math
|
||||
import pathlib
|
||||
import statistics
|
||||
import sys
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
|
||||
from py_markdown_table.markdown_table import markdown_table
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
@@ -32,12 +39,28 @@ parser.add_argument(
|
||||
dest="issue_comment",
|
||||
help="GitHub issue comment defining the regression benchmark profile to use",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--results-file",
|
||||
dest="results_file",
|
||||
help="Path to the results file containing the baseline and last run results",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--generate-report",
|
||||
dest="generate_report",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Generate markdown report of the regression check",
|
||||
)
|
||||
|
||||
COMMENT_IDENTIFIER = "/bench"
|
||||
|
||||
SECONDS_IN_NANO = 1e9
|
||||
MILLISECONDS_IN_NANO = 1e6
|
||||
MICROSECONDS_IN_NANO = 1e3
|
||||
|
||||
CWD = pathlib.Path(__file__).parent
|
||||
REPO_ROOT = CWD.parent
|
||||
PROFILE_DEFINITION_PATH = CWD.joinpath("regression.toml")
|
||||
REPO_ROOT = CWD.parent.parent
|
||||
PROFILE_DEFINITION_PATH = CWD.parent.joinpath("regression.toml")
|
||||
BENCH_TARGETS_PATH = REPO_ROOT.joinpath("tfhe-benchmark/Cargo.toml")
|
||||
# Files generated after parsing an issue comment
|
||||
GENERATED_COMMANDS_PATH = CWD.joinpath("perf_regression_generated_commands.json")
|
||||
@@ -305,6 +328,8 @@ class ProfileDefinition:
|
||||
case TfheBackend.Hpu:
|
||||
features.extend(["hpu", "hpu-v80"])
|
||||
|
||||
features.append("pbs-stats")
|
||||
|
||||
return features
|
||||
|
||||
def generate_cargo_commands(self):
|
||||
@@ -415,12 +440,312 @@ def write_backend_config_to_file(backend, profile):
|
||||
:return:
|
||||
"""
|
||||
for filepart, content in [("backend", backend), ("profile", profile)]:
|
||||
pathlib.Path(f"ci/perf_regression_slab_{filepart}_config.txt").write_text(
|
||||
CWD.joinpath(f"perf_regression_slab_{filepart}_config.txt").write_text(
|
||||
f"{content}\n"
|
||||
)
|
||||
|
||||
|
||||
# TODO Perform regression computing by providing a file containing results from database that would be parsed
|
||||
def write_regression_config_to_file(tfhe_rs_backend, regression_profile):
|
||||
"""
|
||||
Write tfhe-rs backend and regression configuration to different files to ease parsing.
|
||||
|
||||
:param backend:
|
||||
:param profile:
|
||||
:return:
|
||||
"""
|
||||
for filepart, content in [
|
||||
("tfhe_rs_backend", tfhe_rs_backend),
|
||||
("selected_profile", regression_profile),
|
||||
]:
|
||||
CWD.joinpath(f"perf_regression_{filepart}_config.txt").write_text(
|
||||
f"{content}\n"
|
||||
)
|
||||
|
||||
|
||||
# Scale factor to improve the signal/noise ratio in anomaly detection.
|
||||
MAJOR_CHANGE_SCALE_FACTOR = 4
|
||||
MINOR_CHANGE_SCALE_FACTOR = 2
|
||||
REGRESSION_REPORT_FILE = CWD.joinpath("regression_report.md")
|
||||
|
||||
|
||||
class PerfChange(enum.StrEnum):
|
||||
NoChange = "no changes"
|
||||
MinorImprovement = "minor improvement"
|
||||
MajorImprovement = "improvement"
|
||||
MinorRegression = "minor regression"
|
||||
MajorRegression = "regression"
|
||||
|
||||
def get_emoji(self):
|
||||
match self:
|
||||
case PerfChange.NoChange:
|
||||
return ":heavy_minus_sign:"
|
||||
case PerfChange.MinorImprovement:
|
||||
return ":white_check_mark:"
|
||||
case PerfChange.MajorImprovement:
|
||||
return ":heavy_check_mark:"
|
||||
case PerfChange.MinorRegression:
|
||||
return ":warning:"
|
||||
case PerfChange.MajorRegression:
|
||||
return ":bangbang:"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OperationPerformance:
|
||||
name: str
|
||||
baseline_mean: float
|
||||
baseline_stdev: float
|
||||
head_branch_value: float
|
||||
change_percentage: float
|
||||
change_type: PerfChange = PerfChange.NoChange
|
||||
|
||||
def __init__(self, name: str, baseline_data: list[float], head_branch_value: float):
|
||||
self.name = name
|
||||
self.baseline_mean = round(statistics.mean(baseline_data), 2)
|
||||
self.baseline_stdev = round(statistics.stdev(baseline_data), 2)
|
||||
self.head_branch_value = head_branch_value
|
||||
|
||||
def compute_change(self):
|
||||
self.change_percentage = round(
|
||||
(self.head_branch_value - self.baseline_mean) / self.baseline_mean * 100, 2
|
||||
)
|
||||
|
||||
major_threshold = MAJOR_CHANGE_SCALE_FACTOR * self.baseline_stdev
|
||||
minor_threshold = MINOR_CHANGE_SCALE_FACTOR * self.baseline_stdev
|
||||
|
||||
if self.head_branch_value > self.baseline_mean + major_threshold:
|
||||
self.change_type = PerfChange.MajorRegression
|
||||
elif self.head_branch_value > self.baseline_mean + minor_threshold:
|
||||
self.change_type = PerfChange.MinorRegression
|
||||
elif (self.head_branch_value < self.baseline_mean + minor_threshold) and (
|
||||
self.head_branch_value > self.baseline_mean - minor_threshold
|
||||
):
|
||||
# Between +/- MINOR_CHANGE_SCALE_FACTOR * Std_dev we consider there is no change
|
||||
self.change_type = PerfChange.NoChange
|
||||
elif self.head_branch_value < self.baseline_mean - major_threshold:
|
||||
self.change_type = PerfChange.MajorImprovement
|
||||
elif self.head_branch_value < self.baseline_mean - minor_threshold:
|
||||
self.change_type = PerfChange.MinorImprovement
|
||||
|
||||
return self.change_percentage, self.change_type
|
||||
|
||||
def change_percentage_as_str(self):
|
||||
if (
|
||||
self.change_type == PerfChange.MajorImprovement
|
||||
or self.change_type == PerfChange.MinorImprovement
|
||||
or (
|
||||
self.change_type == PerfChange.NoChange
|
||||
and (self.head_branch_value < self.baseline_mean)
|
||||
)
|
||||
):
|
||||
sign = ""
|
||||
else:
|
||||
# Minus sign is already embedded in the float value.
|
||||
sign = "+"
|
||||
|
||||
return f"{sign}{self.change_percentage}%"
|
||||
|
||||
|
||||
def convert_value_to_readable_text(value_nanos: float, max_digits=3):
|
||||
"""
|
||||
Convert timing in nanoseconds to the highest unit usable.
|
||||
|
||||
:param value_nanos: timing value
|
||||
:param max_digits: number of digits to keep in the final representation of the value
|
||||
|
||||
:return: human-readable value with unit as :class:`str`
|
||||
"""
|
||||
if value_nanos > SECONDS_IN_NANO:
|
||||
converted_parts = (value_nanos / SECONDS_IN_NANO), "s"
|
||||
elif value_nanos > MILLISECONDS_IN_NANO:
|
||||
converted_parts = (value_nanos / MILLISECONDS_IN_NANO), "ms"
|
||||
elif value_nanos > MICROSECONDS_IN_NANO:
|
||||
converted_parts = (value_nanos / MICROSECONDS_IN_NANO), "us"
|
||||
else:
|
||||
converted_parts = value_nanos, "ns"
|
||||
|
||||
power_of_10 = math.floor(math.log10(converted_parts[0]))
|
||||
rounding_digit = max_digits - (power_of_10 + 1)
|
||||
if converted_parts[0] >= 100.0:
|
||||
rounding_digit = None
|
||||
|
||||
return f"{round(converted_parts[0], rounding_digit)} {converted_parts[1]}"
|
||||
|
||||
|
||||
def check_performance_changes(results_file: pathlib.Path):
|
||||
"""
|
||||
Check if any operation has regressed compared to the base branch.
|
||||
|
||||
Results file must be in JSON format with the following structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"backend": "<tfhe-rs_backend>",
|
||||
"profile": "<regression_profile>",
|
||||
"operation": [
|
||||
{
|
||||
"name": "<operation_name>",
|
||||
"bit_size": <int>,
|
||||
"params": "<parameters_alias>",
|
||||
|
||||
"results": {
|
||||
"base": {
|
||||
"name": "<base_branche_name>",
|
||||
"data": [
|
||||
<float>,
|
||||
<float>,
|
||||
...,
|
||||
]
|
||||
},
|
||||
"head": {
|
||||
"name": "<dev_branch_name>",
|
||||
"value": <float>
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
:param results_file: path to the result file
|
||||
:type results_file: pathlib.Path
|
||||
|
||||
:return: :class:`list` of :class:`OperationPerformance`
|
||||
"""
|
||||
changes = []
|
||||
|
||||
results = json.loads(results_file.read_text())
|
||||
for operation in results["operations"]:
|
||||
op_name = operation["name"]
|
||||
try:
|
||||
baseline_data = operation["results"]["base"]["data"]
|
||||
except KeyError:
|
||||
raise KeyError(
|
||||
f"no base branch data found in results file for '{op_name}' operation"
|
||||
)
|
||||
|
||||
try:
|
||||
head_branch_value = operation["results"]["head"]["value"]
|
||||
except KeyError:
|
||||
raise KeyError(
|
||||
f"no head branch value found in results file for '{op_name}' operation"
|
||||
)
|
||||
|
||||
op_perf = OperationPerformance(op_name, baseline_data, head_branch_value)
|
||||
op_perf.compute_change()
|
||||
changes.append(op_perf)
|
||||
|
||||
return changes, results["backend"], results["profile"]
|
||||
|
||||
|
||||
OPERATION_HEADER = "Operation"
|
||||
CURRENT_VALUE_HEADER = "Current (ms)"
|
||||
BASELINE_VALUE_HEADER = "Baseline (ms)"
|
||||
BASELINE_STDDEV_HEADER = "Baseline Stddev (ms)"
|
||||
CHANGE_HEADER = "Change (%)"
|
||||
STATUS_HEADER = "Status"
|
||||
|
||||
|
||||
def generate_regression_report(
|
||||
ops_performances: list[OperationPerformance], backend: str, profile: str
|
||||
):
|
||||
"""
|
||||
Generate a regression report in Markdown format and write it to a specified file.
|
||||
|
||||
This function analyzes performance data for various operations and generates a Markdown
|
||||
formatted report summarizing the performance. It highlights any major regressions by
|
||||
providing detailed data about the affected operations and overall benchmark results.
|
||||
Additionally, it includes configuration details, such as the backend and regression profile
|
||||
used in the analysis.
|
||||
|
||||
:param ops_performances: A list of performance data for operations to be analyzed
|
||||
and summarized.
|
||||
:type ops_performances: list[OperationPerformance]
|
||||
:param backend: The backend being used in the performance analysis.
|
||||
:type backend: str
|
||||
:param profile: The profile identifying the regression analysis configuration.
|
||||
:type profile: str
|
||||
"""
|
||||
full_data = [
|
||||
{
|
||||
OPERATION_HEADER: op.name,
|
||||
CURRENT_VALUE_HEADER: convert_value_to_readable_text(op.head_branch_value),
|
||||
BASELINE_VALUE_HEADER: convert_value_to_readable_text(op.baseline_mean),
|
||||
BASELINE_STDDEV_HEADER: convert_value_to_readable_text(op.baseline_stdev),
|
||||
CHANGE_HEADER: op.change_percentage_as_str(),
|
||||
STATUS_HEADER: " ".join((op.change_type.get_emoji(), op.change_type.value)),
|
||||
}
|
||||
for op in ops_performances
|
||||
]
|
||||
full_array_markdown = (
|
||||
markdown_table(full_data)
|
||||
.set_params(row_sep="markdown", quote=False)
|
||||
.get_markdown()
|
||||
)
|
||||
|
||||
regression_data = []
|
||||
for op in ops_performances:
|
||||
if op.change_type != PerfChange.MajorRegression:
|
||||
continue
|
||||
|
||||
regression_data.append(
|
||||
{
|
||||
OPERATION_HEADER: op.name,
|
||||
CURRENT_VALUE_HEADER: convert_value_to_readable_text(
|
||||
op.head_branch_value
|
||||
),
|
||||
BASELINE_VALUE_HEADER: convert_value_to_readable_text(op.baseline_mean),
|
||||
CHANGE_HEADER: op.change_percentage_as_str(),
|
||||
}
|
||||
)
|
||||
|
||||
comment_body = []
|
||||
|
||||
if regression_data:
|
||||
regression_array_markdown = (
|
||||
markdown_table(regression_data)
|
||||
.set_params(row_sep="markdown", quote=False)
|
||||
.get_markdown()
|
||||
)
|
||||
|
||||
regression_details = [
|
||||
"> [!CAUTION]",
|
||||
"> Performances for some operations have regressed compared to the base branch.",
|
||||
"", # Add a newline since tables cannot be rendered in markdown note.
|
||||
regression_array_markdown,
|
||||
"", # Add a newline to avoid rendering the next line of text into the array.
|
||||
]
|
||||
comment_body.extend(regression_details)
|
||||
else:
|
||||
comment_body.append("No performance regression detected. :tada:")
|
||||
|
||||
comment_body.append(
|
||||
"\n".join(
|
||||
[
|
||||
"Configuration",
|
||||
f"* backend: `{backend}`",
|
||||
f"* regression-profile: `{profile}`",
|
||||
"",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
all_results_details = [
|
||||
"<details>",
|
||||
"<summary><strong>View All Benchmarks</strong></summary>",
|
||||
"",
|
||||
full_array_markdown,
|
||||
"</details>",
|
||||
]
|
||||
comment_body.extend(all_results_details)
|
||||
|
||||
formatted_text = "\n".join(comment_body)
|
||||
|
||||
try:
|
||||
REGRESSION_REPORT_FILE.write_text(formatted_text)
|
||||
except Exception as err:
|
||||
print(f"failed to write regression report (error: {err})")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
@@ -449,8 +774,27 @@ if __name__ == "__main__":
|
||||
write_backend_config_to_file(
|
||||
definition.slab_backend, definition.slab_profile
|
||||
)
|
||||
write_regression_config_to_file(
|
||||
definition.backend, definition.regression_profile
|
||||
)
|
||||
except Exception as err:
|
||||
print(f"failed to write commands/env to file (error:{err})")
|
||||
sys.exit(3)
|
||||
elif args.command == "check_regression":
|
||||
pass
|
||||
results_file = args.results_file
|
||||
if not results_file:
|
||||
print(
|
||||
f"cannot run `{args.command}` command: please specify the results file path with `--results-file` argument"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
results_file_path = pathlib.Path(results_file)
|
||||
perf_changes, backend, profile = check_performance_changes(results_file_path)
|
||||
|
||||
if args.generate_report:
|
||||
try:
|
||||
generate_regression_report(perf_changes, backend, profile)
|
||||
except Exception:
|
||||
sys.exit(4)
|
||||
|
||||
# TODO Add unittests primarly to check if commands and env generated are correct.
|
||||
1
ci/perf_regression/requirements.txt
Normal file
1
ci/perf_regression/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
py-markdown-table==1.3.0
|
||||
@@ -42,20 +42,25 @@
|
||||
# See ci/slab.toml file to have the list of all supported profiles.
|
||||
|
||||
[gpu.default]
|
||||
target.integer-bench = ["mul", "div"]
|
||||
target.hlapi-dex = ["dex_swap"]
|
||||
target.integer = ["mul"]
|
||||
target.hlapi-dex = ["dex::swap_claim::no_cmux"]
|
||||
slab.backend = "hyperstack"
|
||||
slab.profile = "single-h100"
|
||||
env.fast_bench = "TRUE"
|
||||
env.bench_param_type = "MULTI_BIT"
|
||||
|
||||
[gpu.multi-h100]
|
||||
target.integer-bench = ["mul", "div"]
|
||||
target.integer = ["mul", "div"]
|
||||
target.hlapi-dex = ["dex_swap"]
|
||||
slab.backend = "hyperstack"
|
||||
slab.profile = "multi-h100"
|
||||
env.fast_bench = "TRUE"
|
||||
env.bench_param_type = "MULTI_BIT"
|
||||
|
||||
[cpu.default]
|
||||
target.integer-bench = ["add_parallelized", "mul_parallelized", "div_parallelized"]
|
||||
target.integer = ["add_parallelized"]
|
||||
target.hlapi-dex = ["dex::swap_claim::no_cmux"]
|
||||
target.hlapi-erc20 = ["transfer::whitepaper"]
|
||||
slab.backend = "aws"
|
||||
slab.profile = "bench"
|
||||
env.fast_bench = "TRUE"
|
||||
|
||||
@@ -18,7 +18,7 @@ user = "ubuntu"
|
||||
|
||||
[backend.aws.bench]
|
||||
region = "eu-west-1"
|
||||
image_id = "ami-0b8fd02b263ebdcf4"
|
||||
image_id = "ami-06e3a9d9b57d38676"
|
||||
instance_type = "hpc7a.96xlarge"
|
||||
user = "ubuntu"
|
||||
|
||||
|
||||
@@ -43,25 +43,25 @@ pbs-stats = ["tfhe/pbs-stats"]
|
||||
zk-pok = ["tfhe/zk-pok"]
|
||||
|
||||
[[bench]]
|
||||
name = "boolean-bench"
|
||||
name = "boolean"
|
||||
path = "benches/boolean/bench.rs"
|
||||
harness = false
|
||||
required-features = ["boolean", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "shortint-bench"
|
||||
name = "shortint"
|
||||
path = "benches/shortint/bench.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "oprf-shortint-bench"
|
||||
name = "shortint-oprf"
|
||||
path = "benches/shortint/oprf.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "glwe_packing_compression-shortint-bench"
|
||||
name = "shortint-glwe_packing_compression"
|
||||
path = "benches/shortint/glwe_packing_compression.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
@@ -91,55 +91,55 @@ harness = false
|
||||
required-features = ["integer", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "glwe_packing_compression-integer-bench"
|
||||
name = "integer-glwe_packing_compression"
|
||||
path = "benches/integer/glwe_packing_compression.rs"
|
||||
harness = false
|
||||
required-features = ["integer", "pbs-stats", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "integer-bench"
|
||||
name = "integer"
|
||||
path = "benches/integer/bench.rs"
|
||||
harness = false
|
||||
required-features = ["integer", "pbs-stats", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "integer-signed-bench"
|
||||
name = "integer-signed"
|
||||
path = "benches/integer/signed_bench.rs"
|
||||
harness = false
|
||||
required-features = ["integer", "pbs-stats", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "zk-pke-bench"
|
||||
name = "integer-zk-pke"
|
||||
path = "benches/integer/zk_pke.rs"
|
||||
harness = false
|
||||
required-features = ["integer", "zk-pok", "pbs-stats", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "ks-bench"
|
||||
name = "core_crypto-ks"
|
||||
path = "benches/core_crypto/ks_bench.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "pbs-bench"
|
||||
name = "core_crypto-pbs"
|
||||
path = "benches/core_crypto/pbs_bench.rs"
|
||||
harness = false
|
||||
required-features = ["boolean", "shortint", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "ks-pbs-bench"
|
||||
name = "core_crypto-ks-pbs"
|
||||
path = "benches/core_crypto/ks_pbs_bench.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
|
||||
[[bench]]
|
||||
name = "modulus_switch_noise_reduction"
|
||||
name = "core_crypto-modulus_switch_noise_reduction"
|
||||
path = "benches/core_crypto/modulus_switch_noise_reduction.rs"
|
||||
harness = false
|
||||
required-features = ["shortint"]
|
||||
|
||||
[[bench]]
|
||||
name = "pbs128-bench"
|
||||
name = "core_crypto-pbs128"
|
||||
path = "benches/core_crypto/pbs128_bench.rs"
|
||||
harness = false
|
||||
required-features = ["shortint", "internal-keycache"]
|
||||
|
||||
Reference in New Issue
Block a user