chore(ci): check for performance regression and create report

After running performances regression benchmarks, a performance
changes checking is executed. It will fetch results data with an
external tool then it will look for anomaly in changes.
Finally it will produce a report as an issue comment with any
anomaly display in a Markdown array. A folded section of the
report message contains all the results from the benchmark.

Note that a fully custom benchmark triggered from an issue comment
would not generate a report. In addition HPU performance
regression benchmark is not supported yet.

.github/workflows/benchmark_perf_regression.yml

@@ -41,7 +41,12 @@ jobs:
       slab-backend: ${{ steps.set_slab_details.outputs.backend }}
       slab-profile: ${{ steps.set_slab_details.outputs.profile }}
       hardware-name: ${{ steps.get_hardware_name.outputs.name }}
+      tfhe-backend: ${{ steps.set_regression_details.outputs.tfhe-backend }}
+      selected-regression-profile: ${{ steps.set_regression_details.outputs.selected-profile }}
       custom-env: ${{ steps.get_custom_env.outputs.custom_env }}
+    permissions:
+      # Needed to react to benchmark command in issue comment
+      issues: write
     steps:
       - name: Checkout tfhe-rs repo
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
@@ -49,23 +54,27 @@ jobs:
           persist-credentials: 'false'
           token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
 
-      - name: Generate cpu benchmarks command from label
+      - name: Generate CPU benchmarks command from label
         if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-cpu'))
         run: |
           echo "DEFAULT_BENCH_OPTIONS=--backend cpu" >> "${GITHUB_ENV}"
 
-      - name: Generate cpu benchmarks command from label
+      - name: Generate GPU benchmarks command from label
         if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-gpu'))
         run: |
           echo "DEFAULT_BENCH_OPTIONS=--backend gpu" >> "${GITHUB_ENV}"
 
       # TODO add support for HPU backend
 
+      - name: Install Python requirements
+        run: |
+          python3 -m pip install -r ci/perf_regression/requirements.txt
+
       - name: Generate cargo commands and env from label
         if: github.event_name == 'pull_request'
         run: |
-          python3 ci/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}"
+          python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}"
-          echo "COMMANDS=$(cat ci/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
+          echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
 
       - name: Dump issue comment into file # To avoid possible code-injection
         if: github.event_name == 'issue_comment'
@@ -77,8 +86,15 @@ jobs:
       - name: Generate cargo commands and env
         if: github.event_name == 'issue_comment'
         run: |
-          python3 ci/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)"
+          python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)"
-          echo "COMMANDS=$(cat ci/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
+          echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}"
+
+      - name: Acknowledge issue comment
+        if: github.event_name == 'issue_comment'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: +1
 
       - name: Set commands output
         id: set_commands
@@ -88,8 +104,8 @@ jobs:
       - name: Set Slab details outputs
         id: set_slab_details
         run: |
-          echo "backend=$(cat ci/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}"
+          echo "backend=$(cat ci/perf_regression/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}"
-          echo "profile=$(cat ci/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}"
+          echo "profile=$(cat ci/perf_regression/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}"
 
       - name: Get hardware name
         id: get_hardware_name
@@ -97,10 +113,16 @@ jobs:
           HARDWARE_NAME=$(python3 ci/hardware_finder.py "${{ steps.set_slab_details.outputs.backend }}" "${{ steps.set_slab_details.outputs.profile }}");
           echo "name=${HARDWARE_NAME}" >> "${GITHUB_OUTPUT}"
 
+      - name: Set regression details outputs
+        id: set_regression_details
+        run: |
+          echo "tfhe-backend=$(cat ci/perf_regression/perf_regression_tfhe_rs_backend_config.txt)" >> "${GITHUB_OUTPUT}"
+          echo "selected-profile=$(cat ci/perf_regression/perf_regression_selected_profile_config.txt)" >> "${GITHUB_OUTPUT}"
+
       - name: Get custom env vars
         id: get_custom_env
         run: |
-          echo "custom_env=$(cat ci/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}"
+          echo "custom_env=$(cat ci/perf_regression/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}"
 
   setup-instance:
     name: benchmark_perf_regression/setup-instance
@@ -134,7 +156,6 @@ jobs:
       - name: Checkout tfhe-rs repo
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
         with:
-          fetch-depth: 0
           persist-credentials: 'false'
           token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
 
@@ -154,14 +175,15 @@ jobs:
       cancel-in-progress: true
     timeout-minutes: 720  # 12 hours
     strategy:
+      fail-fast: false
       max-parallel: 1
       matrix:
         command: ${{ fromJson(needs.prepare-benchmarks.outputs.commands) }}
     steps:
-      - name: Checkout tfhe-rs repo with tags
+      - name: Checkout tfhe-rs repo
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
         with:
-          fetch-depth: 0
+          fetch-depth: 0  # Needed to get commit hash
           persist-credentials: 'false'
           token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
 
@@ -222,7 +244,7 @@ jobs:
 
       - name: Run regression benchmarks
         run: |
-          make  BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom
+          make BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom
         env:
           BENCH_COMMAND: ${{ matrix.command }}
 
@@ -231,6 +253,7 @@ jobs:
           python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \
           --database tfhe_rs \
           --hardware "${HARDWARE_NAME}" \
+          --backend "${TFHE_BACKEND}" \
           --project-version "${COMMIT_HASH}" \
           --branch "${REF_NAME}" \
           --commit-date "${COMMIT_DATE}" \
@@ -238,15 +261,18 @@ jobs:
           --walk-subdirs \
           --name-suffix regression \
           --bench-type "${BENCH_TYPE}"
+  
+          echo "RESULTS_FILE_SHA=$(sha256sum "${RESULTS_FILENAME}" | cut -d " " -f1)" >> "${GITHUB_ENV}"
         env:
-          REF_NAME: ${{ github.ref_name }}
-          BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }}
           HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
+          TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
+          REF_NAME: ${{ github.head_ref || github.ref_name }}
+          BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
-          name: ${{ github.sha }}_regression
+          name: ${{ github.sha }}_regression_${{ env.RESULTS_FILE_SHA }} # RESULT_FILE_SHA is needed to avoid collision between matrix.command runs
           path: ${{ env.RESULTS_FILENAME }}
 
       - name: Send data to Slab
@@ -258,9 +284,81 @@ jobs:
           JOB_SECRET: ${{ secrets.JOB_SECRET }}
           SLAB_URL: ${{ secrets.SLAB_URL }}
 
+  check-regressions:
+    name: benchmark_perf_regression/check-regressions
+    needs: [ prepare-benchmarks, regression-benchmarks ]
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to write a comment in a pull-request
+      pull-requests: write
+      # Needed to set up Python dependencies
+      contents: read
+    env:
+      REF_NAME: ${{ github.head_ref || github.ref_name }}
+    steps:
+      - name: Checkout tfhe-rs repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          persist-credentials: 'false'
+          token: ${{ secrets.REPO_CHECKOUT_TOKEN }}
+
+      - name: Install recent Python
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        with:
+          python-version: '3.12'
+
+      - name: Fetch data
+        run: |
+          python3 -m pip install -r ci/data_extractor/requirements.txt
+          python3 ci/data_extractor/src/data_extractor.py regression_data \
+          --generate-regression-json \
+          --regression-profiles ci/regression.toml \
+          --regression-selected-profile "${REGRESSION_PROFILE}" \
+          --backend "${TFHE_BACKEND}" \
+          --hardware "${HARDWARE_NAME}" \
+          --branch "${REF_NAME}" \
+          --time-span-days 60
+        env:
+          REGRESSION_PROFILE: ${{ needs.prepare-benchmarks.outputs.selected-regression-profile }}
+          TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }}
+          HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }}
+          DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATABASE_HOST }}
+          DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATABASE_USER }}
+          DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
+
+      - name: Generate regression report
+        run: |
+          python3 -m pip install -r ci/perf_regression/requirements.txt
+          python3 ci/perf_regression/perf_regression.py check_regression \
+          --results-file regression_data.json \
+          --generate-report
+
+      - name: Write report in pull-request
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          issue-number: ${{ github.event.pull_request.number || github.event.issue.number }}
+          body-path: ci/perf_regression/regression_report.md
+
+  comment-on-failure:
+    name: benchmark_perf_regression/comment-on-failure
+    needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
+    runs-on: ubuntu-latest
+    if: ${{ failure() && github.event_name == 'issue_comment' }}
+    continue-on-error: true
+    permissions:
+      # Needed to write a comment in a pull-request
+      pull-requests: write
+    steps:
+      - name: Write failure message
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            :x: Performance regression benchmark failed ([workflow run](${{ env.ACTION_RUN_URL }}))
+
   slack-notify:
     name: benchmark_perf_regression/slack-notify
-    needs: [ prepare-benchmarks, setup-instance, regression-benchmarks ]
+    needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ]
     runs-on: ubuntu-latest
     if: ${{ failure() }}
     continue-on-error: true
@@ -268,10 +366,8 @@ jobs:
       - name: Send message
         uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661
         env:
-          SLACK_COLOR: ${{ needs.regression-benchmarks.result }}
+          SLACK_COLOR: failure
-          SLACK_MESSAGE: "Performance regression benchmarks finished with status: ${{ needs.regression-benchmarks.result }}. (${{ env.ACTION_RUN_URL }})"
+          SLACK_MESSAGE: "Performance regression benchmarks failed. (${{ env.ACTION_RUN_URL }})"
-
-  # TODO Add job for regression calculation
 
   teardown-instance:
     name: benchmark_perf_regression/teardown-instance

Makefile

@@ -1336,28 +1336,28 @@ print_doc_bench_parameters:
 bench_integer: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-bench \
+	--bench integer \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_signed_integer # Run benchmarks for signed integer
 bench_signed_integer: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-signed-bench \
+	--bench integer-signed \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_gpu # Run benchmarks for integer on GPU backend
 bench_integer_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-bench \
+	--bench integer \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_signed_integer_gpu # Run benchmarks for signed integer on GPU backend
 bench_signed_integer_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-signed-bench \
+	--bench integer-signed \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_hpu # Run benchmarks for integer on HPU backend
@@ -1366,28 +1366,28 @@ bench_integer_hpu: install_rs_check_toolchain
 	export V80_PCIE_DEV=${V80_PCIE_DEV}; \
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-bench \
+	--bench integer \
 	--features=integer,internal-keycache,pbs-stats,hpu,hpu-v80 -p tfhe-benchmark -- --quick
 
 .PHONY: bench_integer_compression # Run benchmarks for unsigned integer compression
 bench_integer_compression: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench	glwe_packing_compression-integer-bench \
+	--bench	integer-glwe_packing_compression \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_compression_gpu
 bench_integer_compression_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench	glwe_packing_compression-integer-bench \
+	--bench	integer-glwe_packing_compression \
 	--features=integer,internal-keycache,gpu,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_zk_gpu
 bench_integer_zk_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench zk-pke-bench \
+	--bench integer-zk-pke \
 	--features=integer,internal-keycache,gpu,pbs-stats,zk-pok -p tfhe-benchmark --
 
 .PHONY: bench_integer_multi_bit # Run benchmarks for unsigned integer using multi-bit parameters
@@ -1395,7 +1395,7 @@ bench_integer_multi_bit: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-bench \
+	--bench integer \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_signed_integer_multi_bit # Run benchmarks for signed integer using multi-bit parameters
@@ -1403,7 +1403,7 @@ bench_signed_integer_multi_bit: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-signed-bench \
+	--bench integer-signed \
 	--features=integer,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_multi_bit_gpu # Run benchmarks for integer on GPU backend using multi-bit parameters
@@ -1411,7 +1411,7 @@ bench_integer_multi_bit_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
 	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-bench \
+	--bench integer \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_signed_integer_multi_bit_gpu # Run benchmarks for signed integer on GPU backend using multi-bit parameters
@@ -1419,14 +1419,14 @@ bench_signed_integer_multi_bit_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=MULTI_BIT \
 	__TFHE_RS_BENCH_OP_FLAVOR=$(BENCH_OP_FLAVOR) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench integer-signed-bench \
+	--bench integer-signed \
 	--features=integer,gpu,internal-keycache,nightly-avx512,pbs-stats -p tfhe-benchmark --
 
 .PHONY: bench_integer_zk # Run benchmarks for integer encryption with ZK proofs
 bench_integer_zk: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench zk-pke-bench \
+	--bench integer-zk-pke \
 	--features=integer,internal-keycache,zk-pok,nightly-avx512,pbs-stats \
 	-p tfhe-benchmark --
 
@@ -1454,56 +1454,56 @@ bench_boolean: install_rs_check_toolchain
 bench_ks: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench ks-bench \
+	--bench core_crypto-ks \
 	--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_ks_gpu # Run benchmarks for keyswitch on GPU backend
 bench_ks_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench ks-bench \
+	--bench core_crypto-ks \
 	--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_pbs # Run benchmarks for PBS
 bench_pbs: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench pbs-bench \
+	--bench core_crypto-pbs \
 	--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_pbs_gpu # Run benchmarks for PBS on GPU backend
 bench_pbs_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_FAST_BENCH=$(FAST_BENCH) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench pbs-bench \
+	--bench core_crypto-pbs \
 	--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_ks_pbs # Run benchmarks for KS-PBS
 bench_ks_pbs: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench ks-pbs-bench \
+	--bench core_crypto-ks-pbs \
 	--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_ks_pbs_gpu # Run benchmarks for KS-PBS on GPU backend
 bench_ks_pbs_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_PARAM_TYPE=$(BENCH_PARAM_TYPE) __TFHE_RS_PARAMS_SET=$(BENCH_PARAMS_SET) __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench ks-pbs-bench \
+	--bench core_crypto-ks-pbs \
 	--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_pbs128 # Run benchmarks for PBS using FFT 128 bits
 bench_pbs128: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench pbs128-bench \
+	--bench core_crypto-pbs128 \
 	--features=boolean,shortint,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 .PHONY: bench_pbs128_gpu # Run benchmarks for PBS using FFT 128 bits on GPU
 bench_pbs128_gpu: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" __TFHE_RS_BENCH_TYPE=$(BENCH_TYPE) \
 	cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
-	--bench pbs128-bench \
+	--bench core_crypto-pbs128 \
 	--features=boolean,shortint,gpu,internal-keycache,nightly-avx512 -p tfhe-benchmark
 
 bench_web_js_api_parallel_chrome: browser_path = "$(WEB_RUNNER_DIR)/chrome/chrome-linux64/chrome"

ci/data_extractor/requirements.txt

@@ -0,0 +1,4 @@
+psycopg2-binary==2.9.9
+py-markdown-table==1.3.0
+svg.py==1.5.0
+black

ci/data_extractor/src/benchmark_specs.py

@@ -0,0 +1,490 @@
+import enum
+
+from exceptions import ParametersFormatNotSupported
+
+
+class Backend(enum.StrEnum):
+    """
+    Represents different types of computation backends used in tfhe-rs.
+    """
+
+    CPU = "cpu"
+    GPU = "gpu"
+
+    @staticmethod
+    def from_str(backend_name):
+        match backend_name.lower():
+            case "cpu":
+                return Backend.CPU
+            case "gpu":
+                return Backend.GPU
+            case _:
+                raise NotImplementedError
+
+
+class Layer(enum.StrEnum):
+    """
+    Represents different types of layers used in tfhe-rs.
+    """
+
+    HLApi = "hlapi"
+    Integer = "integer"
+    Shortint = "shortint"
+    CoreCrypto = "core_crypto"
+
+    @staticmethod
+    def from_str(layer_name):
+        match layer_name.lower():
+            case "hlapi":
+                return Layer.HLApi
+            case "integer":
+                return Layer.Integer
+            case "shortint":
+                return Layer.Shortint
+            case "core_crypto":
+                return Layer.CoreCrypto
+            case _:
+                raise NotImplementedError
+
+
+class RustType(enum.Enum):
+    """
+    Represents different integer Rust types used in tfhe-rs.
+    """
+
+    FheUint2 = 2
+    FheUint4 = 4
+    FheUint8 = 8
+    FheUint16 = 16
+    FheUint32 = 32
+    FheUint64 = 64
+    FheUint128 = 128
+    FheUint256 = 256
+
+
+ALL_RUST_TYPES = [
+    RustType.FheUint2,
+    RustType.FheUint4,
+    RustType.FheUint8,
+    RustType.FheUint16,
+    RustType.FheUint32,
+    RustType.FheUint64,
+    RustType.FheUint128,
+    RustType.FheUint256,
+]
+
+
+class CoreCryptoOperation(enum.StrEnum):
+    """
+    Represents different core crypto operations performed in tfhe-rs.
+    The values are the ones displayed in the public benchmarks documentation.
+    """
+
+    KeySwitch = "KS"
+    PBS = "PBS"
+    MultiBitPBS = "MB-PBS"
+    KeyswitchPBS = "KS - PBS"
+    KeySwitchMultiBitPBS = "KS - MB-PBS"
+
+    @staticmethod
+    def from_str(operation_name):
+        match operation_name.lower():
+            case "keyswitch":
+                return CoreCryptoOperation.KeySwitch
+            case "pbs_mem_optimized":
+                return CoreCryptoOperation.PBS
+            case "multi_bit_pbs" | "multi_bit_deterministic_pbs":
+                return CoreCryptoOperation.MultiBitPBS
+            case "ks_pbs":
+                return CoreCryptoOperation.KeyswitchPBS
+            case "multi_bit_ks_pbs" | "multi_bit_deterministic_ks_pbs":
+                return CoreCryptoOperation.KeySwitchMultiBitPBS
+            case _:
+                raise NotImplementedError(
+                    f"core crypto operation '{operation_name}' not supported yet"
+                )
+
+    def display_name(self):
+        """
+        Return the human-friendly name recorded for a given operation.
+        This name is parameter-independent.
+
+        :return: The name as recorded by tfhe-benchmark crate
+        :rtype: str
+        """
+        match self:
+            case CoreCryptoOperation.KeySwitch:
+                return "ks"
+            case CoreCryptoOperation.PBS:
+                return "pbs"
+            case CoreCryptoOperation.MultiBitPBS:
+                return "pbs"
+            case CoreCryptoOperation.KeyswitchPBS:
+                return "ks-pbs"
+            case CoreCryptoOperation.KeySwitchMultiBitPBS:
+                return "ks-pbs"
+            case _:
+                raise NotImplementedError(
+                    f"display name for {self} not implemented yet"
+                )
+
+
+class SignFlavor(enum.StrEnum):
+    """
+    Represents the sign of integer benchmarks.
+    """
+
+    Signed = "signed"
+    Unsigned = "unsigned"
+
+
+class OperandType(enum.StrEnum):
+    """
+    Represents the type of operand use in a benchmark.
+    Ciphertext means encrypted-encrypted operation.
+    PlainText means encrypted-plaintext operation.
+    """
+
+    CipherText = "CipherText"  # As represented in the database
+    PlainText = "PlainText"
+
+
+class PBSKind(enum.StrEnum):
+    """
+    Represents the kind of parameter set used for Programmable Bootstrapping operation.
+    """
+
+    Classical = "classical"
+    MultiBit = "multi_bit"
+    Any = "any"  # Special variant used when user doesn't care about the PBS kind
+
+    @staticmethod
+    def from_str(pbs_name):
+        match pbs_name.lower():
+            case "classical":
+                return PBSKind.Classical
+            case "multi_bit":
+                return PBSKind.MultiBit
+            case "any":
+                return PBSKind.Any
+            case _:
+                raise NotImplementedError
+
+
+class NoiseDistribution(enum.StrEnum):
+    """
+    Represents the noise distribution used in the parameter set.
+    """
+
+    Gaussian = "gaussian"
+    TUniform = "tuniform"
+
+    @staticmethod
+    def from_str(distrib_name):
+        match distrib_name.lower():
+            case "gaussian":
+                return NoiseDistribution.Gaussian
+            case "tuniform":
+                return NoiseDistribution.TUniform
+            case _:
+                raise NotImplementedError(
+                    f"noise distribution '{distrib_name}' not supported yet"
+                )
+
+
+class ErrorFailureProbability(enum.IntEnum):
+    """
+    Represents the error failure probability associated with a parameter set.
+    """
+
+    TWO_MINUS_40 = 40
+    TWO_MINUS_64 = 64
+    TWO_MINUS_128 = 128
+
+    @staticmethod
+    def from_param_name(name):
+        parts = name.split("_")
+        for part in parts:
+            if not part.startswith("2M"):
+                continue
+
+            match int(part.lstrip("2M")):
+                case 40:
+                    return ErrorFailureProbability.TWO_MINUS_40
+                case 64:
+                    return ErrorFailureProbability.TWO_MINUS_64
+                case 128:
+                    return ErrorFailureProbability.TWO_MINUS_128
+                case _:
+                    raise NotImplementedError(
+                        f"error failure probability '{part}' not supported yet"
+                    )
+        else:
+            raise ValueError(f"Could not find p-fail value in '{name}'")
+
+    def to_str(self):
+        match self:
+            case ErrorFailureProbability.TWO_MINUS_40:
+                return "2M40"
+            case ErrorFailureProbability.TWO_MINUS_64:
+                return "2M64"
+            case ErrorFailureProbability.TWO_MINUS_128:
+                return "2M128"
+            case _:
+                raise ValueError(
+                    f"error failure probability str conversion '{self}' not supported yet"
+                )
+
+
+class ParamsDefinition:
+    """
+    Represents a parameter definition for specific cryptographic settings.
+
+    The class `ParamsDefinition` is designed to parse and manage parameters derived from
+    a specified parameter name. It facilitates comparison, representation, and hashing of
+    parameter configurations. Parameters related to message size, carry size, noise characteristics,
+    failure probabilities, and other functionalities are extracted and stored for streamlined
+    usage across the system.
+
+    :param param_name: The raw name of the parameter set.
+    :type param_name: str
+    """
+
+    def __init__(self, param_name: str):
+        self.message_size = None
+        self.carry_size = None
+        self.pbs_kind = None
+        self.grouping_factor = None
+        self.noise_distribution = None
+        self.atomic_pattern = None
+        self.p_fail = None
+        self.version = None
+        self.details = {}
+
+        self._parse_param_name(param_name)
+
+    def __eq__(self, other):
+        return (
+            self.message_size == other.message_size
+            and self.carry_size == other.carry_size
+            and self.pbs_kind == other.pbs_kind
+            and self.grouping_factor == other.grouping_factor
+            and self.noise_distribution == other.noise_distribution
+            and self.atomic_pattern == other.atomic_pattern
+            and self.p_fail == other.p_fail
+            and self.version == other.version
+            and self.details == other.details
+        )
+
+    def __lt__(self, other):
+
+        return (
+            self.message_size < other.message_size
+            and self.carry_size < other.carry_size
+            and self.p_fail < other.p_fail
+        )
+
+    def __hash__(self):
+        return hash(
+            (
+                self.message_size,
+                self.carry_size,
+                self.pbs_kind,
+                self.grouping_factor,
+                self.noise_distribution,
+                self.atomic_pattern,
+                self.p_fail,
+                self.version,
+            )
+        )
+
+    def __repr__(self):
+        return f"ParamsDefinition(message_size={self.message_size}, carry_size={self.carry_size}, pbs_kind={self.pbs_kind}, grouping_factor={self.grouping_factor}, noise_distribution={self.noise_distribution}, atomic_pattern={self.atomic_pattern}, p_fail={self.p_fail}, version={self.version}, details={self.details})"
+
+    def _parse_param_name(self, param_name: str) -> None:
+        split_params = param_name.split("_")
+
+        if split_params[0].startswith("V"):
+            minor_version = split_params.pop(1)
+            major_version = split_params.pop(0).strip("V")
+            self.version = major_version + "_" + minor_version
+
+        # Use to know if a parameter set is a derivative of a compute one (e.g. compression parameters)
+        params_variation_parts = []
+        for part in split_params:
+            if part == "PARAM":
+                self.details["variation"] = "_".join(params_variation_parts)
+                break
+
+            params_variation_parts.append(part)
+
+        try:
+            self.p_fail = ErrorFailureProbability.from_param_name(param_name)
+            pfail_index = split_params.index(self.p_fail.to_str())
+        except ValueError or NotImplementedError:
+            # Default error probability may not be shown in the name
+            self.p_fail = ErrorFailureProbability.TWO_MINUS_128
+            pfail_index = None
+
+        if pfail_index:
+            noise_distribution_index = pfail_index - 1
+            self.noise_distribution = NoiseDistribution.from_str(
+                split_params[noise_distribution_index]
+            )
+        else:
+            # Default noise distribution may not be shown in the name
+            self.noise_distribution = NoiseDistribution.TUniform
+            noise_distribution_index = None
+
+        try:
+            self.message_size = int(split_params[split_params.index("MESSAGE") + 1])
+            carry_size_index = split_params.index("CARRY") + 1
+            self.carry_size = int(split_params[carry_size_index])
+            self.atomic_pattern = "_".join(
+                split_params[carry_size_index + 1 : noise_distribution_index]
+            )
+        except ValueError:
+            # Might be a Boolean parameters set
+            raise ParametersFormatNotSupported(param_name)
+
+        try:
+            if noise_distribution_index:
+                self.atomic_pattern = "_".join(
+                    split_params[carry_size_index + 1 : noise_distribution_index]
+                )
+            else:
+                self.atomic_pattern = "_".join(split_params[carry_size_index + 1 :])
+        except ValueError:
+            # Might be a Boolean parameters set
+            raise ParametersFormatNotSupported(param_name)
+
+        try:
+            self.details["trailing_details"] = "_".join(split_params[pfail_index + 1 :])
+        except IndexError:
+            # No trailing details
+            pass
+
+        try:
+            # This is a multi-bit parameters set
+            self.grouping_factor = int(split_params[split_params.index("GROUP") + 1])
+            self.pbs_kind = PBSKind.MultiBit
+        except ValueError:
+            # This is a classical parameters set
+            self.pbs_kind = PBSKind.Classical
+
+
+class BenchDetails:
+    """
+    Represents the details of a benchmark test for different layers.
+
+    This class is designed to parse benchmark information, extract meaningful
+    details such as operation name, parameters, and relevant configuration
+    based on the layer type. It allows for comparison between different benchmark
+    details and outputs structured information for representation or hashing.
+
+    :param layer: The layer the benchmark pertains to.
+    :type layer: Layer
+    :param bench_full_name: Complete name of the benchmark operation.
+    :type bench_full_name: str
+    :param bit_size: The bit size associated with the benchmark.
+    :type bit_size: int
+    """
+
+    def __init__(self, layer: Layer, bench_full_name: str, bit_size: int):
+        self.layer = layer
+
+        self.operation_name = None
+        self.bit_size = bit_size
+        self.params = None
+        # Only relevant for Integer layer
+        self.sign_flavor = None
+        # Only relevant for HLApi layer
+        self.rust_type = None
+
+        self.parse_test_name(bench_full_name)
+
+    def __repr__(self):
+        return f"BenchDetails(layer={self.layer.value}, operation_name={self.operation_name}, bit_size={self.bit_size}, params={self.params}, {self.sign_flavor})"
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __eq__(self, other):
+        return (
+            self.layer == other.layer
+            and self.operation_name == other.operation_name
+            and self.bit_size == other.bit_size
+            and self.params == other.params
+            and self.sign_flavor == other.sign_flavor
+            and self.rust_type == other.rust_type
+        )
+
+    def __hash__(self):
+        return hash(
+            (
+                self.layer,
+                self.operation_name,
+                self.bit_size,
+                self.params,
+                self.rust_type,
+                self.sign_flavor,
+            )
+        )
+
+    def parse_test_name(self, name) -> None:
+        """
+        Parse test name to split relevant parts.
+
+        :param name: The raw test name.
+        :type name: str
+
+        :return: None
+        """
+        parts = name.split("::")
+
+        for part in parts:
+            if "PARAM" in part:
+                self.params = part.partition("_mean")[0]
+                break
+
+        match self.layer:
+            case Layer.Integer:
+                op_name_index = 2 if parts[1] == "cuda" else 1
+                if parts[op_name_index] == "signed":
+                    op_name_index += 1
+                    self.sign_flavor = SignFlavor.Signed
+                    self.operation_name = parts[op_name_index]
+                elif parts[op_name_index] == "unsigned":
+                    # This is a pattern used by benchmark run on CUDA
+                    op_name_index += 1
+                    self.sign_flavor = SignFlavor.Unsigned
+                    self.operation_name = parts[op_name_index]
+                else:
+                    self.sign_flavor = SignFlavor.Unsigned
+                    self.operation_name = parts[op_name_index]
+                    self.params = parts[op_name_index + 1]
+                    if "compression" in parts[op_name_index]:
+                        self.rust_type = "_".join(
+                            (parts[op_name_index], parts[-1].split("_")[0])
+                        )
+            case Layer.CoreCrypto:
+                self.operation_name = parts[2] if parts[1] == "cuda" else parts[1]
+            case Layer.HLApi:
+                if parts[1] == "cuda":
+                    self.operation_name = "::".join(parts[2:-1])
+                else:
+                    self.operation_name = "::".join(parts[1:-1])
+                self.rust_type = parts[-1].partition("_mean")[0]
+            case _:
+                raise NotImplementedError(
+                    f"layer '{self.layer}' not supported yet for name parsing"
+                )
+
+    def get_params_definition(self) -> ParamsDefinition:
+        """
+        Returns the definition of parameters based on the current instance's parameters.
+
+        :return: A ParamsDefinition object that encapsulates the parameter definition.
+        :rtype: ParamsDefinition
+        """
+        return ParamsDefinition(self.params)

ci/data_extractor/src/config.py

@@ -0,0 +1,43 @@
+import argparse
+import pathlib
+
+from benchmark_specs import Backend, Layer, PBSKind
+
+
+class UserConfig:
+    """
+    Manages the configuration provided by the user input.
+
+    This class encapsulates the user-provided configuration data necessary for execution.
+    It sets up various attributes based on the input arguments, converting or transforming
+    values as needed to ensure compatibility and correct behavior within the system.
+
+    :param input_args: The input arguments provided by the user.
+    :type input_args: argparse.Namespace
+    """
+
+    def __init__(self, input_args: argparse.Namespace):
+        self.output_file = input_args.output_file
+
+        self.database = input_args.database
+        self.backend = Backend.from_str(input_args.backend.lower())
+        self.hardware = input_args.hardware  # This input is case-sensitive
+
+        self.head_branch = input_args.branch.lower()
+        self.base_branch = input_args.base_branch.lower()
+
+        self.project_version = input_args.project_version
+
+        self.bench_date = input_args.bench_date
+        self.time_span_days = input_args.time_span_days
+
+        self.layer = Layer.from_str(input_args.layer.lower())
+        self.pbs_kind = PBSKind.from_str(input_args.pbs_kind)
+        self.grouping_factor = input_args.grouping_factor
+
+        self.regression_selected_profile = input_args.regression_selected_profile
+        self.regression_profiles_path = (
+            pathlib.Path(input_args.regression_profiles)
+            if input_args.regression_profiles
+            else None
+        )

ci/data_extractor/src/connector.py

@@ -0,0 +1,292 @@
+import configparser
+import datetime
+import os
+import pathlib
+
+import psycopg2
+from benchmark_specs import BenchDetails, Layer, OperandType, PBSKind
+from config import UserConfig
+from exceptions import NoDataFound
+
+
+class PostgreConfig:
+    """
+    Represents the configuration to connect to a PostgreSQL database.
+
+    This class is designed to manage and load PostgreSQL database credentials
+    from a configuration file and override them with environment variables if available.
+
+    :param path: Path to the configuration file.
+    :type path: str, optional
+
+    :ivar host: Host address of the PostgreSQL database.
+    :type host: str
+    :ivar user: Username for connecting to the PostgreSQL database.
+    :type user: str
+    :ivar password: Password for connecting to the PostgreSQL database.
+    :type password: str
+    """
+
+    def __init__(self, path: str = None):
+        self.host = None
+        self.user = None
+        self.password = None
+
+        if path:
+            self._from_config_file(path)
+
+        self._override_with_env()
+
+    def _from_config_file(self, path):
+        """
+        Parse configuration file containing credentials to Postgre database.
+
+        :param path: path to the configuration file as :class:`str`
+
+        :return: parsed file as :class:`configparser.ConfigParser`
+        """
+        path = pathlib.Path(path)
+        conf = configparser.ConfigParser()
+        conf.read(path)
+
+        for option_name in ("user", "password", "host"):
+            try:
+                attr_value = conf.get("postgre", option_name)
+            except ValueError:
+                print(f"Cannot find mandatory option '{option_name}' in config file")
+                raise
+
+            setattr(self, option_name, attr_value)
+
+    def _override_with_env(self):
+        self.host = os.environ.get("DATA_EXTRACTOR_DATABASE_HOST", self.host)
+        self.user = os.environ.get("DATA_EXTRACTOR_DATABASE_USER", self.user)
+        self.password = os.environ.get(
+            "DATA_EXTRACTOR_DATABASE_PASSWORD", self.password
+        )
+
+
+class PostgreConnector:
+    def __init__(self, conf: PostgreConfig):
+        """
+        Initializes the class with a configuration object.
+
+        :param conf: The configuration object used to set up
+                     the connection or relevant settings.
+        :type conf: PostgreConfig
+        """
+        self.conf = conf
+        self._conn = None
+
+    def __del__(self):
+        if self._conn:
+            # Avoid dandling connection on server side.
+            self._conn.close()
+
+    def connect_to_database(self, dbname: str):
+        """
+        Create a connection to a Postgre instance.
+
+        :param dbname: Name of the database to connect to.
+        :type dbname: str
+        """
+        print("Connecting to database... ", end="", flush=True)
+        start_date = datetime.datetime.now()
+
+        try:
+            conn = psycopg2.connect(
+                dbname=dbname,
+                user=self.conf.user,
+                password=self.conf.password,
+                host=self.conf.host,
+            )
+        except psycopg2.Error as err:
+            print(f"Failed to connect to Postgre database")
+            raise
+        else:
+            elapsed = (datetime.datetime.now() - start_date).total_seconds()
+            print(f"connected in {elapsed}s")
+
+        self._conn = conn
+
+    def close(self):
+        """
+        Close the connection to the database.
+
+        :return: None
+        """
+        self._conn.close()
+        self._conn = None
+
+    def fetch_benchmark_data(
+        self,
+        user_config: UserConfig,
+        operand_type: OperandType = None,
+        operation_filter: list = None,
+        layer: Layer = None,
+        branch: str = None,
+        name_suffix: str = "_mean_avx512",
+        last_value_only: bool = True,
+    ) -> dict[BenchDetails, list[int]]:
+        """
+        Fetches benchmark data from the database based on various filtering criteria.
+
+        Filters and constructs a query based on the provided user configuration and optional
+        parameters such as operand type, operation filters, layer, and others. It retrieves
+        specific benchmark metrics like test names, bit sizes, metric values, and their last
+        inserted values. The data is fetched for a given hardware, backend, branch, and project
+        version, or within a computed time range.
+
+        :param user_config: User's configuration specifying backend, hardware, branch, layer,
+            project version, PBS kind, benchmark date, and time span.
+        :type user_config: UserConfig
+        :param operand_type: Optional operand type filter.
+        :type operand_type: OperandType, optional
+        :param operation_filter: Optional list of operation name filters for partial test name matching.
+        :type operation_filter: list, optional
+        :param layer: Optional tfhe-rs layer filter
+        :type layer: Layer, optional
+        :param branch: Optional branch filter, defaulting to the user's head branch if not specified.
+        :type branch: str, optional
+        :param name_suffix: Suffix to match the test names, defaulting to "_mean_avx512".
+        :type name_suffix: str, optional
+        :param last_value_only: A flag indicating whether to fetch only the most recent metric value for each benchmark.
+        :type last_value_only: bool
+
+        :return: Fetched benchmark data filtered and formatted as per the query.
+        :rtype: dict[BenchDetails, list[int]]
+        """
+        backend = user_config.backend
+        branch = branch.lower() if branch else user_config.head_branch
+        hardware = user_config.hardware
+        layer = layer if layer else user_config.layer
+        version = user_config.project_version
+        pbs_kind = user_config.pbs_kind
+
+        timestamp_range_end = user_config.bench_date
+        timestamp = datetime.datetime.fromisoformat(timestamp_range_end)
+        time_span_delta = datetime.timedelta(days=user_config.time_span_days)
+        timestamp_range_start = datetime.datetime.isoformat(timestamp - time_span_delta)
+        if layer == Layer.CoreCrypto:
+            # Before this date we used another name format that is not parsable by this tool.
+            cutoff_date = "2024-02-28T00:00:00"
+            timestamp_range_start = max(cutoff_date, timestamp_range_start)
+
+        filters = list()
+
+        filters.append(f"h.name = '{hardware}'")
+        filters.append(f"bk.name = '{backend}'")
+        filters.append(f"b.name = '{branch}'")
+
+        name_suffix = f"\\{name_suffix}"
+        if backend == "cpu":
+            filters.append(f"test.name LIKE '{layer}::%{name_suffix}'")
+        elif backend == "gpu":
+            filters.append(f"test.name LIKE '{layer}::cuda::%{name_suffix}'")
+
+        if version:
+            filters.append(f"pv.name = '{version}'")
+        else:
+            filters.append(f"m.insert_time >= '{timestamp_range_start}'")
+            filters.append(f"m.insert_time <= '{timestamp_range_end}'")
+
+        # First iteration to fetch only default operations
+        filters.append("test.name NOT SIMILAR TO '%(smart|unchecked)_%'")
+        match pbs_kind:
+            case PBSKind.Classical:
+                filters.append(
+                    "p.crypto_parameters_alias NOT SIMILAR TO '%_MULTI_BIT_%'"
+                )
+            case PBSKind.MultiBit:
+                filters.append("p.crypto_parameters_alias SIMILAR TO '%_MULTI_BIT_%'")
+            case PBSKind.Any:
+                # No need to add a filter
+                pass
+
+        if operand_type:
+            filters.append(f"p.operand_type = '{operand_type.value}'")
+
+        if operation_filter:
+            conditions = [
+                f"test.name LIKE '%::{op_name}::%'" for op_name in operation_filter
+            ]
+            filters.append("({})".format(" OR ".join(conditions)))
+
+        # Throughput is not supported yet
+        filters.append("test.name NOT SIMILAR TO '%::throughput::%'")
+
+        select_parts = (
+            "SELECT",
+            "test.name as test,",
+            "p.bit_size as bit_size,",
+            "m.value as value,",
+            "m.insert_time,",
+            "LAST_VALUE (value)",
+            "OVER (PARTITION BY test.name ORDER BY m.insert_time RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) last_value",
+            "FROM benchmark.metrics as m",
+            "LEFT JOIN benchmark.project_version as pv ON m.project_version_id = pv.id",
+            "LEFT JOIN benchmark.hardware as h ON m.hardware_id = h.id",
+            "LEFT JOIN benchmark.backend as bk ON m.backend_id = bk.id",
+            "LEFT JOIN benchmark.branch as b ON b.id = m.branch_id",
+            "LEFT JOIN benchmark.test as test ON test.id = m.test_id",
+            "LEFT JOIN benchmark.parameters as p ON p.id = m.parameters_id",
+        )
+
+        sql_string = format(
+            "{} WHERE {} GROUP BY test, bit_size, value, m.insert_time ORDER BY m.insert_time DESC"
+        ).format(" ".join(select_parts), " AND ".join(filters))
+
+        return self._fetch_data(
+            sql_string,
+            version,
+            timestamp_range_start,
+            timestamp_range_end,
+            hardware,
+            layer,
+            last_value_only,
+        )
+
+    def _fetch_data(
+        self,
+        sql_string,
+        version,
+        timestamp_range_start,
+        timestamp_range_end,
+        hw,
+        layer: Layer,
+        last_value_only: bool,
+    ) -> dict[BenchDetails, list[int]]:
+        with self._conn.cursor() as curs:
+            start = datetime.datetime.now()
+            print(f"Fetching data (hardware: {hw}, layer: {layer.value})...", end="")
+
+            curs.execute(sql_string)
+            lines = curs.fetchall()
+
+            end = datetime.datetime.now()
+            print(f"done in {(end - start).total_seconds()}s")
+
+            results = dict()
+
+            if not lines:
+                if version:
+                    msg = f"no data found under commit hash '{version}'"
+                else:
+                    msg = f"no data found in date range [{timestamp_range_start}, {timestamp_range_end}]"
+                raise NoDataFound(msg)
+
+            for line in lines:
+                bit_width = line[1]
+
+                bench_details = BenchDetails(layer, line[0], bit_width)
+                value = line[-1] if last_value_only else line[-3]
+                try:
+                    timings = results[bench_details]
+                    if last_value_only:
+                        continue
+                    else:
+                        timings.append(value)
+                except KeyError:
+                    results[bench_details] = [value]
+
+        return results

ci/data_extractor/src/data_extractor.py

@@ -0,0 +1,403 @@
+"""
+data_extractor
+--------------
+
+Extract benchmarks results from Zama PostgreSQL instance.
+It will output the filtered results in a file formatted as CSV.
+
+PostgreSQL connection configuration can be passed through a configuration file or via environment variables.
+When using the environment variables, make sure to set the following ones:
+ * DATA_EXTRACTOR_DATABASE_HOST
+ * DATA_EXTRACTOR_DATABASE_USER
+ * DATA_EXTRACTOR_DATABASE_PASSWORD
+
+Note that if provided, environment variables will take precedence over the configuration file.
+"""
+
+import argparse
+import datetime
+import formatter
+import sys
+from formatter import (CSVFormatter, GenericFormatter, MarkdownFormatter,
+                       SVGFormatter)
+
+import config
+import connector
+import regression
+from benchmark_specs import Backend, Layer, OperandType, PBSKind, RustType
+
+import utils
+
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group()
+parser.add_argument(
+    "output_file", help="File storing parsed results (with no extension)"
+)
+parser.add_argument(
+    "-c" "--config-file",
+    dest="config_file",
+    help="Location of configuration file containing credentials to connect to "
+    "PostgreSQL instance",
+)
+parser.add_argument(
+    "--bench-date",
+    dest="bench_date",
+    default=datetime.datetime.now().isoformat(),
+    help=(
+        "Last insertion date to look for in the database,"
+        " formatted as ISO 8601 timestamp YYYY-MM-DDThh:mm:ss"
+    ),
+)
+parser.add_argument(
+    "-d",
+    "--database",
+    dest="database",
+    default="tfhe_rs",
+    help="Name of the database used to store results",
+)
+group.add_argument(
+    "-w",
+    "--hardware",
+    dest="hardware",
+    default="hpc7a.96xlarge",
+    help="Hardware reference used to perform benchmark",
+)
+group.add_argument(
+    "--hardware-comp",
+    dest="hardware_comp",
+    help="Comma separated values of hardware to compare. "
+    "The first value would be chosen as baseline.",
+)
+parser.add_argument(
+    "-V", "--project-version", dest="project_version", help="Commit hash reference"
+)
+parser.add_argument(
+    "-b",
+    "--branch",
+    dest="branch",
+    default="main",
+    help="Git branch name on which benchmark was performed",
+)
+parser.add_argument(
+    "--base-branch",
+    dest="base_branch",
+    default="main",
+    help="Git base branch name on which benchmark history can be fetched",
+)
+parser.add_argument(
+    "--backend",
+    dest="backend",
+    choices=["cpu", "gpu"],
+    default="cpu",
+    help="Backend on which benchmarks have run",
+)
+parser.add_argument(
+    "--tfhe-rs-layer",
+    dest="layer",
+    default="integer",
+    help="Layer of the tfhe-rs library to filter against",
+)
+parser.add_argument(
+    "--pbs-kind",
+    dest="pbs_kind",
+    choices=["classical", "multi_bit", "any"],
+    default="classical",
+    help="Kind of PBS to look for",
+)
+parser.add_argument(
+    "--grouping-factor",
+    dest="grouping_factor",
+    type=int,
+    choices=[2, 3, 4],
+    help="Grouping factor used in multi-bit parameters set",
+)
+parser.add_argument(
+    "--time-span-days",
+    dest="time_span_days",
+    type=int,
+    default=30,
+    help="Numbers of days prior of `bench_date` we search for results in the database",
+)
+parser.add_argument(
+    "--regression-profiles",
+    dest="regression_profiles",
+    help="Path to file containing regression profiles formatted as TOML",
+)
+parser.add_argument(
+    "--regression-selected-profile",
+    dest="regression_selected_profile",
+    help="Regression profile to select from the regression profiles file to filter out database results",
+)
+exclusive_generation_group = parser.add_mutually_exclusive_group()
+exclusive_generation_group.add_argument(
+    "--generate-markdown",
+    dest="generate_markdown",
+    action="store_true",
+    help="Generate Markdown array",
+)
+exclusive_generation_group.add_argument(
+    "--generate-svg",
+    dest="generate_svg",
+    action="store_true",
+    help="Generate SVG table formatted like ones in tfhe-rs documentation",
+)
+exclusive_generation_group.add_argument(
+    "--generate-svg-from-markdown",
+    dest="generate_svg_from_file",
+    help="Generate SVG table formatted like ones in tfhe-rs documentation from a Markdown table",
+)
+exclusive_generation_group.add_argument(
+    "--generate-regression-json",
+    dest="generate_regression_json",
+    action="store_true",
+    help="Generate JSON file with regression data with all the results from base branch and the lastest results of the development branch",
+)
+
+
+def generate_svg_from_file(
+    user_config: config.UserConfig, layer: Layer, input_file: str
+):
+    """
+    Generates an SVG file based on a given formatted array in Markdown file.
+
+    :param user_config: An instance of the UserConfig class, used to manage
+        configuration details like backend, PBS kind, and output file paths.
+    :type user_config: config.UserConfig
+    :param layer: The layer information used in SVG formatting and
+        generation.
+    :type layer: Layer
+    :param input_file: File path of the input Markdown file to be converted to
+        SVG format.
+    :type input_file: str
+
+    :return: None
+    """
+    utils.write_to_svg(
+        SVGFormatter(
+            layer,
+            user_config.backend,
+            user_config.pbs_kind,
+        ).generate_svg_table_from_markdown_file(input_file),
+        user_config.output_file,
+    )
+
+
+def perform_hardware_comparison(
+    user_config: config.UserConfig,
+    layer: Layer,
+):
+    """
+    Perform a hardware comparison by fetching benchmark data, computing
+    comparisons, and generating CSV outputs for each hardware configuration. It
+    outputs both raw data and gain-based analysis for comparison between
+    reference and target hardware.
+
+    :param user_config: An instance of the UserConfig class, used to manage
+        configuration details like backend, PBS kind, and output file paths.
+    :type user_config: config.UserConfig
+    :param layer: The layer object containing specific information required for
+        formatting and processing benchmark data.
+    :type layer: Layer
+
+    :return: None
+    """
+    results = []
+
+    for hw in hardware_list:
+        try:
+            res = conn.fetch_benchmark_data(user_config, operand_type)
+        except RuntimeError as err:
+            print(f"Failed to fetch benchmark data: {err}")
+            sys.exit(2)
+
+        results.append(res)
+
+        output_filename = "".join(
+            [user_config.output_file, "_", hw, "_", operand_type.lower(), ".csv"]
+        )
+        csv_formatter = CSVFormatter(layer, user_config.backend, user_config.pbs_kind)
+        formatted_data = csv_formatter.format_data(
+            res, utils.convert_value_to_readable_text
+        )
+        utils.write_to_csv(
+            csv_formatter.generate_csv(formatted_data),
+            output_filename,
+        )
+
+    gains_results = formatter.compute_comparisons(*results)
+    reference_hardware = hardware_list[0]
+    for i, hw in enumerate(hardware_list[1:]):
+        output_filename = "".join(
+            [
+                user_config.output_file,
+                "_",
+                operand_type.lower(),
+                "_",
+                reference_hardware,
+                "_",
+                hw,
+                "_gains.csv",
+            ]
+        )
+        csv_formatter = CSVFormatter(layer, user_config.backend, user_config.pbs_kind)
+        formatted_data = csv_formatter.format_data(
+            gains_results[i],
+            utils.convert_gain_to_text,
+        )
+        utils.write_to_csv(
+            csv_formatter.generate_csv(formatted_data),
+            output_filename,
+        )
+
+
+def perform_data_extraction(
+    user_config: config.UserConfig,
+    layer: Layer,
+    operand_type: OperandType,
+    output_filename: str,
+    generate_markdown: bool = False,
+    generate_svg: bool = False,
+):
+    """
+    Extracts, formats, and processes benchmark data for a specified operand type and
+    saves the results into various file formats such as CSV, Markdown, or SVG based
+    on user configuration.
+
+    :param user_config: An instance of the UserConfig class, used to manage
+        configuration details like backend, PBS kind, and output file paths.
+    :type user_config: config.UserConfig
+    :param layer: Layer object specifying the granularity and context of the
+        operand processing.
+    :type layer: Layer
+    :param operand_type: Type of operand data for which the benchmarks are
+        extracted and processed.
+    :type operand_type: OperandType
+    :param output_filename: The base filename for the output files where results
+        will be saved.
+    :type output_filename: str
+    :param generate_markdown: Boolean flag indicating whether to generate an
+        output file in Markdown (.md) format.
+    :type generate_markdown: bool
+    :param generate_svg: Boolean flag indicating whether to generate an output
+        file in SVG (.svg) format.
+    :type generate_svg: bool
+    :return: None
+    """
+    try:
+        res = conn.fetch_benchmark_data(user_config, operand_type)
+    except RuntimeError as err:
+        print(f"Failed to fetch benchmark data: {err}")
+        sys.exit(2)
+
+    generic_formatter = GenericFormatter(
+        layer, user_config.backend, user_config.pbs_kind, user_config.grouping_factor
+    )
+    formatted_results = generic_formatter.format_data(
+        res,
+        utils.convert_value_to_readable_text,
+    )
+
+    file_suffix = f"_{operand_type.lower()}"
+    filename = utils.append_suffix_to_filename(output_filename, file_suffix, ".csv")
+
+    utils.write_to_csv(
+        CSVFormatter(layer, user_config.backend, user_config.pbs_kind).generate_csv(
+            formatted_results
+        ),
+        filename,
+    )
+
+    generic_arrays = generic_formatter.generate_array(
+        formatted_results,
+        operand_type,
+        excluded_types=[RustType.FheUint2, RustType.FheUint4, RustType.FheUint256],
+    )
+
+    for array in generic_arrays:
+        metadata_suffix = ""
+        if array.metadata:
+            for key, value in array.metadata.items():
+                metadata_suffix += f"_{key}_{value}"
+
+        current_suffix = file_suffix + metadata_suffix
+
+        if generate_markdown:
+            filename = utils.append_suffix_to_filename(
+                output_filename, current_suffix, ".md"
+            )
+
+            data_formatter = MarkdownFormatter(
+                layer, user_config.backend, user_config.pbs_kind
+            )
+
+            utils.write_to_markdown(
+                data_formatter.generate_markdown_array(array),
+                filename,
+            )
+        elif generate_svg:
+            filename = utils.append_suffix_to_filename(
+                output_filename, current_suffix, ".svg"
+            )
+
+            data_formatter = SVGFormatter(
+                layer, user_config.backend, user_config.pbs_kind
+            )
+
+            utils.write_to_svg(
+                data_formatter.generate_svg_table(
+                    array,
+                ),
+                filename,
+            )
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    user_config = config.UserConfig(args)
+    layer = user_config.layer
+
+    if args.generate_svg_from_file:
+        generate_svg_from_file(user_config, layer, args.generate_svg_from_file)
+        sys.exit(0)
+
+    try:
+        postgre_config = connector.PostgreConfig(args.config_file)
+        conn = connector.PostgreConnector(postgre_config)
+        conn.connect_to_database(user_config.database)
+    except Exception:
+        sys.exit(1)
+
+    if args.generate_regression_json:
+        try:
+            regression.perform_regression_json_generation(conn, user_config)
+        except RuntimeError as err:
+            print(f"Failed to perform performance regression JSON: {err}")
+            sys.exit(2)
+        else:
+            sys.exit(0)
+
+    hardware_list = (
+        args.hardware_comp.lower().split(",") if args.hardware_comp else None
+    )
+
+    for operand_type in (OperandType.CipherText, OperandType.PlainText):
+        if hardware_list:
+            perform_hardware_comparison(user_config, layer)
+
+            if args.generate_markdown:
+                print("Markdown generation is not supported with comparisons")
+            continue
+
+        if layer == Layer.CoreCrypto and operand_type == OperandType.PlainText:
+            continue
+
+        perform_data_extraction(
+            user_config,
+            layer,
+            operand_type,
+            user_config.output_file,
+            generate_markdown=args.generate_markdown,
+            generate_svg=args.generate_svg,
+        )
+
+    conn.close()

ci/data_extractor/src/exceptions.py

@@ -0,0 +1,20 @@
+class NoDataFound(RuntimeError):
+    """
+    Indicates that no data was found when an operation or search was performed.
+
+    This exception should be raised to signal that a requested operation could not
+    complete because the required data was unavailable. It is typically used in
+    cases where returning an empty result might not be appropriate, and an
+    explicit notice of failure is required.
+    """
+
+    pass
+
+
+class ParametersFormatNotSupported(Exception):
+    """
+    Exception raised for unsupported parameter formats.
+    """
+
+    def __init__(self, param_name):
+        super().__init__(f"Parameters format '{param_name}' not supported.")

ci/data_extractor/src/formatter.py

@@ -0,0 +1,940 @@
+import collections
+import copy
+import enum
+import pathlib
+import xml.dom.minidom
+from collections.abc import Callable
+
+import svg
+from benchmark_specs import (ALL_RUST_TYPES, Backend, BenchDetails,
+                             CoreCryptoOperation, ErrorFailureProbability,
+                             Layer, NoiseDistribution, OperandType, PBSKind,
+                             RustType)
+from py_markdown_table.markdown_table import markdown_table
+
+
+def compute_comparisons(*results):
+    """
+    Compute gains for data in ``results``. The first element is considered as the baseline.
+
+    :param results: :class:`list` of results
+
+    :return: :class:`list` of gains with length of ``results`` minus one (baseline is not sent back)
+    """
+    gains = []
+    baseline = results[0]
+
+    for compared_results in results[1:]:
+        for key, base_value in baseline.items():
+            try:
+                compared_value = compared_results[key]
+            except KeyError:
+                # Ignore missing entries
+                continue
+
+            gain = round((base_value - compared_value) * 100 / base_value, 2)
+            compared_results[key] = gain
+
+        gains.append(compared_results)
+
+    return gains
+
+
+class OperationDisplayName(enum.StrEnum):
+    Negation = "Negation (-)"
+    AddSub = "Add / Sub (+,-)"
+    Mul = "Mul (x)"
+    EqualNotEqual = "Equal / Not Equal (eq, ne)"
+    Comparisons = "Comparisons (ge, gt, le, lt)"
+    MaxMin = "Max / Min (max, min)"
+    Bitwise = "Bitwise operations (&, \\|, ^)"
+    Div = "Div  (/)"
+    Rem = "Rem  (%)"
+    DivRem = "Div / Rem  (/, %)"
+    Shifts = "Left / Right Shifts (<<, >>)"
+    Rotates = "Left / Right Rotations (left_rotate, right_rotate)"
+    LeadingTrailing = "Leading / Trailing zeros/ones"
+    Log2 = "Log2"
+    Select = "Select"
+
+
+class BenchArray:
+    def __init__(self, array, layer, metadata: dict = None):
+        self.array = array
+        self.layer = layer
+        self.metadata = metadata
+
+    def __repr__(self):
+        return f"BenchArray(layer={self.layer}, metadata={self.metadata})"
+
+
+class GenericFormatter:
+    def __init__(
+        self,
+        layer: Layer,
+        backend: Backend,
+        pbs_kind: PBSKind,
+        grouping_factor: int = None,
+    ):
+        """
+        Generic formatter for a given specified layer, backend, and PBS kind.
+
+        :param layer: Represents the layer configuration for the object.
+        :type layer: Layer
+        :param backend: Specifies the backend system to be used.
+        :type backend: Backend
+        :param pbs_kind: Specifies the type or kind of PBS (Private Backend System).
+        :type pbs_kind: PBSKind
+        :param grouping_factor: Specifies the multi-bit PBS grouping factor to use a filter.
+        :type grouping_factor: int, optional
+        """
+        self.layer = layer
+        self.backend = backend
+        self.pbs_kind = pbs_kind
+        self.requested_grouping_factor = grouping_factor
+
+    def set_grouping_factor(self, grouping_factor: int):
+        """
+        Sets the grouping factor for a computation or operation.
+
+        This method allows configuration of the requested grouping factor
+        value, which can affect how operations using multi-bit parameters set are processed.
+
+        :param grouping_factor: The desired grouping factor to use.
+        :type grouping_factor: int
+        """
+        self.requested_grouping_factor = grouping_factor
+
+    def format_data(
+        self, data: dict[BenchDetails : list[int]], conversion_func: Callable
+    ):
+        """
+        Formats data based on the specified layer and applies a conversion function to
+        transform the data.
+
+        The method determines the data formatting logic by matching the
+        current layer of the object and invokes the appropriate specific layer-related
+        formatting function.
+
+        :param data: A dictionary where the keys are instances of `BenchDetails` and
+            the values are lists of integers, representing the benchmark data to be
+            formatted.
+        :type data: dict[BenchDetails : list[int]]
+        :param conversion_func: A callable function that will be applied to transform
+            the data values based on the specific layer requirements.
+        :type conversion_func: Callable
+
+        :return: The formatted data results after applying layer and conversion logic.
+        :rtype: Any
+
+        :raises NotImplementedError: Raised when the specified layer is unsupported.
+        """
+        match self.layer:
+            case Layer.Integer:
+                return self._format_integer_data(data, conversion_func)
+            case Layer.CoreCrypto:
+                return self._format_core_crypto_data(data, conversion_func)
+            case _:
+                raise NotImplementedError(f"layer '{self.layer}' not supported yet")
+
+    @staticmethod
+    def _format_integer_data(data: dict[BenchDetails : list[int]], conversion_func):
+        formatted = collections.defaultdict(
+            lambda: {
+                2: "N/A",
+                8: "N/A",
+                16: "N/A",
+                32: "N/A",
+                64: "N/A",
+                128: "N/A",
+                256: "N/A",
+            }
+        )
+        for details, timings in data.items():
+            test_name = "_".join((details.sign_flavor.value, details.operation_name))
+            bit_width = details.bit_size
+            value = conversion_func(timings[-1])
+
+            if bit_width == 40:
+                # Ignore this width as it's not displayed publicly.
+                continue
+
+            formatted[test_name][bit_width] = value
+
+        return formatted
+
+    @staticmethod
+    def _format_core_crypto_data(data: dict[BenchDetails : list[int]], conversion_func):
+        params_set = set()
+        for details in data:
+            try:
+                params_set.add(details.get_params_definition())
+            except Exception:
+                # Might be a Boolean parameters set, ignoring
+                continue
+
+        params_set = sorted(params_set)
+
+        formatted = collections.defaultdict(
+            lambda: {params: "N/A" for params in params_set}
+        )
+        for details, timings in data.items():
+            try:
+                reduced_params = details.get_params_definition()
+            except Exception:
+                # Might be a Boolean parameters set, ignoring
+                continue
+
+            test_name = details.operation_name
+            value = conversion_func(timings[-1])
+            formatted[test_name][reduced_params] = value
+
+        return formatted
+
+    def generate_array(
+        self,
+        data,
+        operand_type: OperandType = None,
+        excluded_types: list[RustType] = None,
+    ) -> list[BenchArray]:
+        """
+        Generates an array of `BenchArray` based on the specified layer and criteria.
+
+        This method takes input data and generates an array of `BenchArray` objects,
+        using the rules defined by the current `layer`. The behavior varies depending
+        on the active layer, and certain types can be explicitly excluded from the
+        generation process.
+
+        :param data: Input data to generate the array from.
+        :type data: Any
+        :param operand_type: Specifies the type of operand to guide the array generation.
+            Defaults to `None`.
+        :type operand_type: OperandType, optional
+        :param excluded_types: A list of `RustType` to exclude from array generation.
+            Defaults to `None`.
+        :type excluded_types: list[RustType], optional
+
+        :return: A list of generated `BenchArray` objects.
+        :rtype: list[BenchArray]
+
+        :raises NotImplementedError: If the current layer is not implemented.
+        """
+        match self.layer:
+            case Layer.Integer:
+                return self._generate_unsigned_integer_array(
+                    data, operand_type, excluded_types
+                )
+            case Layer.CoreCrypto:
+                return self._generate_core_crypto_showcase_arrays(data)
+            case _:
+                raise NotImplementedError
+
+    def _generate_unsigned_integer_array(
+        self,
+        data,
+        operand_type: OperandType = None,
+        excluded_types: list[RustType] = None,
+    ):
+        match operand_type:
+            case OperandType.CipherText:
+                prefix = "unsigned"
+            case OperandType.PlainText:
+                prefix = "unsigned_scalar"
+
+        match self.backend:
+            case Backend.CPU:
+                operations = [
+                    f"{prefix}_neg_parallelized",
+                    f"{prefix}_add_parallelized",
+                    f"{prefix}_mul_parallelized",
+                    f"{prefix}_eq_parallelized",
+                    f"{prefix}_gt_parallelized",
+                    f"{prefix}_max_parallelized",
+                    f"{prefix}_bitand_parallelized",
+                    f"{prefix}_div_rem_parallelized",
+                    f"{prefix}_left_shift_parallelized",
+                    f"{prefix}_rotate_left_parallelized",
+                    f"{prefix}_leading_zeros_parallelized",
+                    f"{prefix}_ilog2_parallelized",
+                    f"{prefix}_if_then_else_parallelized",
+                ]
+            case Backend.GPU:
+                match operand_type:
+                    case OperandType.CipherText:
+                        prefix = "cuda"
+                    case OperandType.PlainText:
+                        prefix = "cuda_scalar"
+
+                operations = [
+                    f"{prefix}_neg",
+                    f"{prefix}_add",
+                    f"{prefix}_mul",
+                    f"{prefix}_eq",
+                    f"{prefix}_gt",
+                    f"{prefix}_max",
+                    f"{prefix}_bitand",
+                    f"{prefix}_div_rem",
+                    f"{prefix}_left_shift",
+                    f"{prefix}_rotate_left",
+                    f"{prefix}_leading_zeros",
+                    f"{prefix}_ilog2",
+                    f"{prefix}_if_then_else",
+                ]
+            case _:
+                raise NotImplementedError(
+                    f"backend '{self.backend}' not supported yet for integer formatting"
+                )
+
+        display_names = [
+            OperationDisplayName.Negation,
+            OperationDisplayName.AddSub,
+            OperationDisplayName.Mul,
+            OperationDisplayName.EqualNotEqual,
+            OperationDisplayName.Comparisons,
+            OperationDisplayName.MaxMin,
+            OperationDisplayName.Bitwise,
+            OperationDisplayName.DivRem,
+            OperationDisplayName.Shifts,
+            OperationDisplayName.Rotates,
+            OperationDisplayName.LeadingTrailing,
+            OperationDisplayName.Log2,
+            OperationDisplayName.Select,
+        ]
+
+        types = ALL_RUST_TYPES.copy()
+        excluded_types = excluded_types if excluded_types is not None else []
+        for excluded in excluded_types:
+            types.remove(excluded)
+
+        first_column_header = "Operation \\ Size"
+
+        # Adapt list to plaintext benchmarks results.
+        if operand_type == OperandType.PlainText:
+            if self.backend == Backend.CPU:
+                div_name = f"{prefix}_div_parallelized"
+                rem_name = f"{prefix}_rem_parallelized"
+            elif self.backend == Backend.GPU:
+                div_name = f"{prefix}_div"
+                rem_name = f"{prefix}_rem"
+
+            operations.insert(8, div_name)
+            operations.insert(9, rem_name)
+            operations.pop(7)  # Remove div_rem_parallelized
+
+            display_names.insert(
+                8,
+                OperationDisplayName.Div,
+            )
+            display_names.insert(
+                9,
+                OperationDisplayName.Rem,
+            )
+            display_names.pop(7)  # Remove Div / Rem
+
+            # Negation operation doesn't exist in plaintext
+            operations.pop(0)
+            display_names.pop(0)
+
+        data_without_excluded_types = copy.deepcopy(data)
+        for v in data_without_excluded_types.values():
+            for excluded in excluded_types:
+                try:
+                    v.pop(excluded.value)
+                except KeyError:
+                    # Type is not contained in the results, ignoring
+                    continue
+
+        filtered_data = filter(lambda t: t in operations, data_without_excluded_types)
+        # Get operation names as key of the dict to ease fetching
+        filtered_data_dict = {
+            item: tuple(data_without_excluded_types[item].values())
+            for item in filtered_data
+        }
+
+        result_lines = []
+        for name, op in zip(display_names, operations):
+            try:
+                line = {first_column_header: name.value}
+                line.update(
+                    {
+                        types[i].name: value
+                        for i, value in enumerate(filtered_data_dict[op])
+                    }
+                )
+                result_lines.append(line)
+            except KeyError:
+                # Operation not found in the results, ignoring this line.
+                print(
+                    f"backend '{self.backend}' could not find operation '{op}' to put in line '{name}'"
+                )
+                continue
+
+        return [
+            BenchArray(result_lines, self.layer),
+        ]
+
+    def _build_results_dict(
+        self,
+        pfails: list[ErrorFailureProbability],
+        noise_distributions: list[NoiseDistribution],
+        operation_displays: list[CoreCryptoOperation],
+        default_precisions: Callable[[], dict],
+    ):
+        results_dict = {}
+
+        for pfail in pfails:
+            for noise in noise_distributions:
+                results_dict[CoreCryptoResultsKey(pfail, noise)] = {
+                    o: default_precisions() for o in operation_displays
+                }
+
+        return results_dict
+
+    def _generate_core_crypto_showcase_arrays(
+        self,
+        data,
+    ):
+        supported_pfails = [
+            ErrorFailureProbability.TWO_MINUS_40,
+            ErrorFailureProbability.TWO_MINUS_64,
+            ErrorFailureProbability.TWO_MINUS_128,
+        ]
+        noise_distributions = [
+            NoiseDistribution.Gaussian,
+            NoiseDistribution.TUniform,
+        ]
+
+        operation_displays = [op.value for op in OPERATIONS_DISPLAYS]
+
+        sorted_results = self._build_results_dict(
+            supported_pfails,
+            noise_distributions,
+            OPERATIONS_DISPLAYS,
+            DEFAULT_CORE_CRYPTO_PRECISIONS,
+        )
+
+        for operation, timings in data.items():
+            try:
+                formatted_name = CoreCryptoOperation.from_str(operation)
+            except NotImplementedError:
+                # Operation is not supported.
+                continue
+
+            for param_definition, value in timings.items():
+                pfail = param_definition.p_fail
+                if pfail not in supported_pfails:
+                    print(f"[{operation}] P-fail '{pfail}' is not supported")
+                    continue
+                noise = param_definition.noise_distribution
+                precision = int(param_definition.message_size) * 2
+                key = CoreCryptoResultsKey(pfail, noise)
+
+                if (
+                    formatted_name == CoreCryptoOperation.MultiBitPBS
+                    or formatted_name == CoreCryptoOperation.KeySwitchMultiBitPBS
+                ) and param_definition.pbs_kind != PBSKind.MultiBit:
+                    # Skip this operation since a multi-bit operation cannot be done with any other parameters type.
+                    continue
+
+                grouping_factor = param_definition.grouping_factor
+                if (
+                    grouping_factor is not None
+                    and grouping_factor != self.requested_grouping_factor
+                ):
+                    continue
+
+                if (
+                    param_definition.details["variation"]
+                    or param_definition.details["trailing_details"]
+                ):
+                    continue
+
+                try:
+                    sorted_results[key][formatted_name][precision] = value
+                except KeyError:
+                    # Operation is not supposed to appear in the formatted array.
+                    continue
+
+        first_column_header = "Operation \\ Precision (bits)"
+
+        arrays = []
+        for key, results in sorted_results.items():
+            array = []
+            for operation, timings in results.items():
+                d = {first_column_header: operation.value}
+                d.update({str(k): v for k, v in timings.items()})
+                array.append(d)
+
+            arrays.append(
+                BenchArray(
+                    array,
+                    self.layer,
+                    metadata={"pfail": key.pfail, "noise": key.noise_distribution},
+                )
+            )
+
+        return arrays
+
+
+# ---------------------------
+# Core_crypto layer constants
+# ---------------------------
+
+OPERATIONS_DISPLAYS = [
+    # CoreCryptoOperation.KeySwitch, # Uncomment this line to get keyswitch in the tables
+    CoreCryptoOperation.PBS,
+    CoreCryptoOperation.MultiBitPBS,
+    CoreCryptoOperation.KeyswitchPBS,
+    CoreCryptoOperation.KeySwitchMultiBitPBS,
+]
+
+DEFAULT_CORE_CRYPTO_PRECISIONS = lambda: {
+    2: "N/A",
+    4: "N/A",
+    6: "N/A",
+    8: "N/A",
+}
+
+
+class CoreCryptoResultsKey:
+    """
+    Representation of a hashable result key for the core_crypto layer.
+
+    :param pfail: Probability of failure associated with the cryptographic result.
+    :type pfail: ErrorFailureProbability
+    :param noise_distribution: Noise distribution parameter linked to the
+        cryptographic result.
+    :type noise_distribution: NoiseDistribution
+    """
+
+    def __init__(
+        self, pfail: ErrorFailureProbability, noise_distribution: NoiseDistribution
+    ):
+        self.pfail = pfail
+        self.noise_distribution = noise_distribution
+
+    def __eq__(self, other):
+        return (
+            self.pfail == other.pfail
+            and self.noise_distribution == other.noise_distribution
+        )
+
+    def __hash__(self):
+        return hash((self.pfail, self.noise_distribution))
+
+    def __repr__(self):
+        return f"CoreCryptoResultsKey(pfail={self.pfail}, noise_distribution={self.noise_distribution})"
+
+
+class CSVFormatter(GenericFormatter):
+    """
+    Formatter to generate CSV content.
+    """
+
+    def generate_csv(self, data: dict[str, collections.defaultdict]) -> list[list]:
+        """
+        Generates a CSV-compatible data structure based on the provided input data and
+        the current layer type. The method processes the input to construct headers and
+        rows suitable for CSV writing.
+
+        :param data: A dictionary where keys represent row identifiers and values
+            are dictionaries of column-value pairs representing the data for
+            each row.
+        :type data: dict[str, collections.defaultdict]
+
+        :return: A list of lists where each sub-list represents a row in the CSV,
+            including the header row.
+        :rtype: list
+
+        :raises NotImplementedError: If the layer type specified in the object's
+            `layer` attribute is unsupported.
+        """
+        headers_values = data.get(list(data)[0]).keys()
+
+        match self.layer:
+            case Layer.Integer:
+                headers = ["Operation \\ Size(bit)", *headers_values]
+            case Layer.CoreCrypto:
+                headers = ["Operation \\ Parameters set", *headers_values]
+            case _:
+                print(
+                    f"tfhe-rs layer '{self.layer}' currently not supported for CSV writing"
+                )
+                raise NotImplementedError
+
+        csv_data = [headers]
+        csv_data.extend(
+            [[key, *list(values_dict.values())] for key, values_dict in data.items()]
+        )
+
+        return csv_data
+
+
+class MarkdownFormatter(GenericFormatter):
+    """
+    Formatter to generate Markdown content.
+    """
+
+    def generate_markdown_array(
+        self,
+        generic_array: BenchArray,
+    ) -> str:
+        """
+        Generates a Markdown representation of the provided generic array.
+
+        :param generic_array: The input array encapsulated in a BenchArray object.
+
+        :return: A Markdown formatted string representing the input array.
+        :rtype: str
+        """
+        md_array = (
+            markdown_table(generic_array.array)
+            .set_params(row_sep="markdown", quote=False, padding_weight="right")
+            .get_markdown()
+        )
+
+        return md_array
+
+
+# -------------
+# SVG constants
+# -------------
+
+BLACK_COLOR = "black"
+WHITE_COLOR = "white"
+LIGHT_GREY_COLOR = "#f3f3f3"
+YELLOW_COLOR = "#fbbc04"
+
+FONT_FAMILY = "Arial"
+FONT_SIZE = 14
+
+BORDER_WIDTH_PIXEL = 2
+# Operation name is always in table first column
+OPERATION_NAME_HORIZONTAL_POSITION = 6
+
+SPECIAL_CHARS_PAIRS = {
+    "&": "&#38;",
+    "<": "&#60;",
+    ">": "&#62;",
+    "\\|": "&#124;",
+}
+
+# -------------
+
+
+class SVGFormatter(GenericFormatter):
+    """
+    Formatter to generate SVG content.
+    """
+
+    @staticmethod
+    def _transform_special_characters(strg: str):
+        for char, replacement in SPECIAL_CHARS_PAIRS.items():
+            if char in strg:
+                strg = strg.replace(char, replacement)
+
+        return strg
+
+    def _build_svg_headers_row(
+        self,
+        layer: Layer,
+        headers,
+        overall_width,
+        row_height,
+        op_name_col_width,
+        per_timing_col_width,
+    ):
+        op_header = headers.pop(0)
+        header_elements = [
+            svg.Rect(
+                x=0, y=0, width=overall_width, height=row_height, fill=BLACK_COLOR
+            ),
+            self._build_svg_text(
+                OPERATION_NAME_HORIZONTAL_POSITION,
+                row_height / 2,
+                op_header,
+                text_anchor="start",
+                fill=WHITE_COLOR,
+                font_weight="bold",
+            ),
+        ]
+
+        for row_idx, type_ident in enumerate(headers):
+            curr_x = op_name_col_width + row_idx * per_timing_col_width
+
+            match layer:
+                case Layer.Integer:
+                    type_name_width = type_ident.strip("FheUint")
+                    header_elements.extend(
+                        [
+                            # Rust type class
+                            self._build_svg_text(
+                                curr_x + per_timing_col_width / 2,
+                                row_height / 3,
+                                "FheUint",
+                                fill=WHITE_COLOR,
+                                font_weight="bold",
+                            ),
+                            # Actual size of the Rust type
+                            self._build_svg_text(
+                                curr_x + per_timing_col_width / 2,
+                                2 * row_height / 3 + 3,
+                                type_name_width,
+                                fill=WHITE_COLOR,
+                                font_weight="bold",
+                            ),
+                        ]
+                    )
+                case Layer.CoreCrypto:
+                    header_elements.append(
+                        # Core_crypto arrays contains only ciphertext modulus size as headers
+                        self._build_svg_text(
+                            curr_x + per_timing_col_width / 2,
+                            row_height / 2,
+                            type_ident,
+                            fill=WHITE_COLOR,
+                            font_weight="bold",
+                        )
+                    )
+                case _:
+                    raise NotImplementedError
+
+        return header_elements
+
+    def _build_svg_timing_row(
+        self,
+        timings_row,
+        row_y_pos,
+        row_height,
+        op_name_col_width,
+        per_timing_col_width,
+    ):
+        timing_elements = []
+        op_name = timings_row.pop(0)
+        timing_elements.append(
+            self._build_svg_text(
+                OPERATION_NAME_HORIZONTAL_POSITION,
+                row_y_pos + row_height / 2,
+                op_name,
+                text_anchor="start",
+            )
+        )
+        for timing_idx, timing in enumerate(timings_row):
+            timing_elements.append(
+                self._build_svg_text(
+                    op_name_col_width
+                    + timing_idx * per_timing_col_width
+                    + per_timing_col_width / 2,
+                    row_y_pos + row_height / 2,
+                    timing,
+                )
+            )
+
+        return timing_elements
+
+    def _build_svg_text(
+        self, x, y, text, text_anchor="middle", fill=BLACK_COLOR, font_weight="normal"
+    ):
+        return svg.Text(
+            x=x,
+            y=y,
+            dominant_baseline="middle",
+            text_anchor=text_anchor,
+            font_family=FONT_FAMILY,
+            font_size=FONT_SIZE,
+            fill=fill,
+            text=text,
+            font_weight=font_weight,
+        )
+
+    def _build_svg_borders(
+        self,
+        overall_width,
+        overall_height,
+        row_height,
+        op_name_col_width,
+        per_timing_col_width,
+        row_count,
+        col_count,
+    ):
+        border_elements = []
+
+        # Horizontal borders, scrolling vertically
+        for row_idx in range(row_count + 2):
+            row_y = row_idx * row_height
+            border_elements.append(
+                svg.Line(
+                    x1=0,
+                    y1=row_y,
+                    x2=overall_width,
+                    y2=row_y,
+                    stroke=WHITE_COLOR,
+                    stroke_width=BORDER_WIDTH_PIXEL,
+                )
+            )
+
+        # Vertical borders, scrolling horizontally
+        # Left border
+        border_elements.append(
+            svg.Line(
+                x1=0,
+                y1=0,
+                x2=0,
+                y2=overall_height,
+                stroke=WHITE_COLOR,
+                stroke_width=BORDER_WIDTH_PIXEL,
+            )
+        )
+        # Timing cols
+        for col_idx in range(col_count + 1):
+            col_x = op_name_col_width + col_idx * per_timing_col_width
+            border_elements.append(
+                svg.Line(
+                    x1=col_x,
+                    y1=0,
+                    x2=col_x,
+                    y2=overall_height,
+                    stroke=WHITE_COLOR,
+                    stroke_width=BORDER_WIDTH_PIXEL,
+                )
+            )
+
+        return border_elements
+
+    def generate_svg_table(self, generic_array: BenchArray) -> str:
+        """
+        Generates an SVG representation of a table from the given `BenchArray` object.
+        This method processes array data to create a visual representation of the
+        provided headers and values as an SVG image, organizing them into rows and
+        columns consistent with the structure of the input.
+
+        :param generic_array: The BenchArray object containing structured data for
+            generating the table.
+        :type generic_array: BenchArray
+
+        :return: An SVG representation of the table as a well-formatted string.
+            The SVG is generated with appropriate dimensions, colors, and styles
+            to ensure a visually clear and consistent layout.
+        :rtype: str
+        """
+        headers = list(generic_array.array[0].keys())
+        ops = generic_array.array[:]
+
+        # TODO Create a class to handle table dimension which will depend on tfhe-rs layer
+        col_count = len(headers) - 1
+        row_count = 1 + len(ops)
+
+        overall_width = int(round(900 / 1.25))
+        row_height = int(round(50 / 1.25))
+        op_name_col_width = int(round(375 / 1.25))
+        per_timing_col_width = (overall_width - op_name_col_width) / col_count
+        overall_height = row_count * row_height
+
+        svg_elements = []
+
+        # Generate headers row
+        svg_elements.extend(
+            self._build_svg_headers_row(
+                self.layer,
+                headers,
+                overall_width,
+                row_height,
+                op_name_col_width,
+                per_timing_col_width,
+            )
+        )
+
+        # Generate operations rectangle
+        yellow_rect_for_op_names = svg.Rect(
+            x=0,
+            y=row_height,
+            width=op_name_col_width,
+            height=overall_height - row_height,
+            fill=YELLOW_COLOR,
+        )
+        svg_elements.append(yellow_rect_for_op_names)
+
+        # Generate timings rectangle
+        grey_rect_for_timings = svg.Rect(
+            x=op_name_col_width,
+            y=row_height,
+            width=overall_width - op_name_col_width,
+            height=overall_height - row_height,
+            fill=LIGHT_GREY_COLOR,
+        )
+        svg_elements.append(grey_rect_for_timings)
+
+        for row_count, row in enumerate(ops):
+            row_y = row_height + row_count * row_height
+            row_split = [
+                self._transform_special_characters(v)
+                for v in filter(None, row.values())
+            ]
+            svg_elements.extend(
+                self._build_svg_timing_row(
+                    row_split,
+                    row_y,
+                    row_height,
+                    op_name_col_width,
+                    per_timing_col_width,
+                )
+            )
+
+        # Generate borders
+        svg_elements.extend(
+            self._build_svg_borders(
+                overall_width,
+                overall_height,
+                row_height,
+                op_name_col_width,
+                per_timing_col_width,
+                row_count,
+                col_count,
+            )
+        )
+
+        canvas = svg.SVG(
+            width="100%",
+            height=overall_height,
+            viewBox=f"0 0 {overall_width} {overall_height}",
+            elements=svg_elements,
+            preserveAspectRatio="meet",
+        )
+
+        dom = xml.dom.minidom.parseString(str(canvas))
+        return dom.toprettyxml()
+
+    def generate_svg_table_from_markdown_file(
+        self,
+        input_filepath: str,
+    ) -> str:
+        """
+        Generates an SVG table from a Markdown file.
+
+        This method reads a Markdown file, parses the table content, and transforms
+        it into an SVG representation. The input Markdown file must have headers
+        defined in the first row and rows of data separated by the "|" character.
+
+        :param input_filepath: The file path to the Markdown file.
+        :type input_filepath: str
+
+        :return: The generated SVG representation of the parsed Markdown table.
+        :rtype: str
+        """
+        with pathlib.Path(input_filepath).open("r") as f:
+            md_lines = f.read().splitlines()
+
+        headers = [h.lstrip().rstrip() for h in md_lines.pop(0).split("|")[1:-1]]
+
+        md_lines.pop(0)  # remove separation line
+        parsed_data = []
+        for line in md_lines:
+            values = [v.lstrip().rstrip() for v in line.split("|")[1:-1]]
+            if len(values) > len(headers):
+                # A '|' character is contained in the operation name
+                values[0] += "".join(["|", values.pop(1)])
+            parsed_data.append({k: v for k, v in zip(headers, values)})
+
+        array = BenchArray(parsed_data, self.layer)
+
+        return self.generate_svg_table(array)

ci/data_extractor/src/regression.py

@@ -0,0 +1,183 @@
+import pathlib
+import sys
+
+import config
+import connector
+from benchmark_specs import Backend, Layer
+from exceptions import NoDataFound
+
+import utils
+
+try:
+    import tomllib  # Python v3.11+
+except ModuleNotFoundError:
+    import pip._vendor.tomli as tomllib  # the same tomllib that's now included in Python v3.11+
+
+
+def generate_json_regression_file(
+    conn: connector.PostgreConnector, ops_filter: dict, user_config: config.UserConfig
+):
+    """
+    Generate a JSON regression file based on benchmark data from specified branches
+    and operations.
+
+    This function extracts benchmark data for a set of operations across HEAD
+    and BASE branches, processes it, and generates a structured JSON output
+    representing the benchmarking comparison.
+
+    :param conn: Connector to the benchmark data source.
+    :type conn: Any
+    :param ops_filter: A dictionary containing operations as keys and their
+        respective filters as values.
+    :type ops_filter: dict
+    :param user_config: A configuration object that contains branch information,
+        backend settings, and output preferences for regression.
+    :type user_config: UserConfig
+
+    :raises NoDataFound: If benchmark data is not found for the HEAD or BASE branch
+        during processing.
+    :raises json.JSONDecodeError: If the final dictionary cannot be converted into
+        JSON format.
+    """
+    error_msg = "Cannot generate JSON regression file (error: no data found on {} branch '{}' (layer: {}, operations: {}))"
+    regression_data = {}
+
+    for bench_target, ops in ops_filter.items():
+        layer_name, _, bench_id = bench_target.partition("-")
+        layer = Layer.from_str(layer_name)
+
+        try:
+            head_branch_data = conn.fetch_benchmark_data(
+                user_config,
+                operation_filter=ops,
+                layer=layer,
+                branch=user_config.head_branch,
+                name_suffix="_mean_regression",
+                last_value_only=True,
+            )
+        except NoDataFound:
+            print(error_msg.format("HEAD", user_config.head_branch, layer, ops))
+            raise
+
+        try:
+            base_branch_data = conn.fetch_benchmark_data(
+                user_config,
+                operation_filter=ops,
+                layer=layer,
+                branch=user_config.base_branch,
+                last_value_only=False,
+            )
+        except NoDataFound:
+            print(error_msg.format("BASE", user_config.base_branch, layer, ops))
+            raise
+
+        for bench_details, values in head_branch_data.items():
+            regression_data[bench_details.operation_name] = {
+                "name": bench_details.operation_name,
+                "bit_size": bench_details.bit_size,
+                "params": bench_details.params,
+                "results": {
+                    "head": {"name": user_config.head_branch, "value": values[0]},
+                },
+            }
+
+        for bench_details, values in base_branch_data.items():
+            try:
+                reg = regression_data[bench_details.operation_name]
+                if (
+                    reg["bit_size"] == bench_details.bit_size
+                    and reg["params"] == bench_details.params
+                ):
+                    reg["results"]["base"] = {
+                        "name": user_config.base_branch,
+                        "data": values,
+                    }
+            except KeyError:
+                # No value exists on the head branch for this key, ignoring the base branch result.
+                continue
+
+    final_dict = {
+        "backend": user_config.backend,
+        "profile": user_config.regression_selected_profile,
+        "operations": list(regression_data.values()),
+    }
+
+    output_file = "".join([user_config.output_file, ".json"])
+    utils.write_to_json(final_dict, output_file)
+
+
+def parse_toml_file(
+    path: str, backend: Backend, profile_name: str
+) -> dict[str, list[str]]:
+    """
+    Parse a TOML file defining regression profiles and return its content as a dictionary.
+
+    :param path: path to TOML file
+    :type path: str
+    :param backend: type of backend used for the benchmarks
+    :type backend: Backend
+    :param profile_name: name of the regression profile
+    :type profile_name: str
+
+    :return: profile content formatted as `{"<benchmark_target>": ["<operation_1>", ...]}`
+    :rtype: dict[str, list[str]]
+    """
+    file_path = pathlib.Path(path)
+    try:
+        return tomllib.loads(file_path.read_text())[backend][profile_name]
+    except tomllib.TOMLDecodeError as err:
+        raise RuntimeError(f"failed to parse definition file (error: {err})")
+    except KeyError:
+        raise RuntimeError(
+            f"failed to find definition profile (error: profile '{profile_name}' cannot be found in '{file_path}')"
+        )
+
+
+def perform_regression_json_generation(
+    conn: connector.PostgreConnector, user_config: config.UserConfig
+):
+    """
+    Generates a JSON file for regression testing based on the selected regression profile.
+
+    This function generates a JSON file with benchmarks for the specified regression profile.
+    The function requires the `regression_selected_profile` attribute of the `user_config` parameter if
+    custom regression profile operations are to be filtered. If no such profile is provided or no profile
+    definitions file is specified, the function terminates the program with an appropriate message.
+
+    :param conn: PostgreConnector object used for generating the JSON regression file.
+    :type conn: Any
+    :param user_config: Configuration object containing user-defined regression profiles and settings.
+    :type user_config: UserConfig
+
+    :raises SystemExit: If no regression profile is selected, or if the regression profiles
+    file is not provided
+    """
+    # TODO Currently this implementation doesn't support custom regression profile.
+    #  It will return strictly the benchmarks results for the operations defined for the selected profile.
+    #  One way to handle that case would be to pass the custom profile as program input (could be either via a string or path containing the profile).
+    if not user_config.regression_selected_profile:
+        print(
+            "Regression generation requires a profile name (see: --regression-selected-profile input argument)"
+        )
+        sys.exit(5)
+    selected_profile = user_config.regression_selected_profile
+
+    operations_filter = {}
+    if user_config.regression_profiles_path:
+        profile_definition = parse_toml_file(
+            user_config.regression_profiles_path, user_config.backend, selected_profile
+        )
+        operations_filter = profile_definition["target"]
+        try:
+            user_config.pbs_kind = profile_definition["env"]["bench_param_type"].lower()
+        except KeyError:
+            # Benchmark parameters type is not declared in the regression definition file
+            # Ignoring
+            pass
+    else:
+        print(
+            "Regression generation requires a profile definitions file (see: --regression-profiles input argument)"
+        )
+        sys.exit(5)
+
+    generate_json_regression_file(conn, operations_filter, user_config)

ci/data_extractor/src/utils.py

@@ -0,0 +1,146 @@
+import csv
+import json
+import math
+import pathlib
+import sys
+
+from benchmark_specs import Layer
+
+SECONDS_IN_NANO = 1e9
+MILLISECONDS_IN_NANO = 1e6
+MICROSECONDS_IN_NANO = 1e3
+
+
+def convert_value_to_readable_text(value: int, max_digits: int = 3) -> str:
+    """
+    Convert timing in nanoseconds to the highest unit usable.
+
+    :param value: timing value
+    :type value: int
+    :param max_digits: number of digits to keep in the final representation of the value
+    :type max_digits: int, optional
+
+    :return: human-readable value with unit
+    :rtype: str
+    """
+    if value > SECONDS_IN_NANO:
+        converted_parts = (value / SECONDS_IN_NANO), "s"
+    elif value > MILLISECONDS_IN_NANO:
+        converted_parts = (value / MILLISECONDS_IN_NANO), "ms"
+    elif value > MICROSECONDS_IN_NANO:
+        converted_parts = (value / MICROSECONDS_IN_NANO), "us"
+    else:
+        converted_parts = value, "ns"
+
+    power_of_10 = math.floor(math.log10(converted_parts[0]))
+    rounding_digit = max_digits - (power_of_10 + 1)
+    if converted_parts[0] >= 100.0:
+        rounding_digit = None
+
+    return f"{round(converted_parts[0], rounding_digit)} {converted_parts[1]}"
+
+
+def convert_gain_to_text(value: float) -> str:
+    """
+    Convert gains as :class:`float` to :class:`str`
+
+    :param value: gain value
+    :type value: float
+
+    :return: gain as text with percentage sign
+    :rtype: str
+    """
+    return f"{value} %" if value < 0 else f"+{value} %"
+
+
+def write_to_csv(lines: list, output_filename: str):
+    """
+    Write data to a CSV file.
+
+    :param lines: formatted data as iterable
+    :type lines: list
+    :param output_filename: filename where data would be written
+    :type output_filename: str
+    """
+    with pathlib.Path(output_filename).open("w") as csv_file:
+        writer = csv.writer(csv_file, delimiter=",")
+        for line in lines:
+            writer.writerow(line)
+
+    print(f"Results written as CSV in '{output_filename}'")
+
+
+def write_to_markdown(lines: str | list[str], output_filename: str):
+    """
+    Write data to a Markdown file.
+
+    :param lines: formatted lines
+    :type lines: str | list[str]
+    :param output_filename: filename where data would be written
+    :type output_filename: str
+    """
+    if type(lines) != str:
+        content = "\n".join(lines) + "\n"
+        pathlib.Path(output_filename).write_text(content)
+    else:
+        pathlib.Path(output_filename).write_text(lines)
+
+    print(f"Results written as Markdown in '{output_filename}'")
+
+
+def write_to_svg(xml_string: str, output_filename: str):
+    """
+    Write XML to a SVG file.
+
+    :param xml_string: XML formatted string
+    :type xml_string: str
+    :param output_filename: filename where data would be written
+    :type output_filename: str
+    """
+    pathlib.Path(output_filename).write_text(xml_string)
+
+    print(f"Results written as SVG in '{output_filename}'")
+
+
+def write_to_json(data: dict, output_filename: str):
+    """
+    Write data to a JSON file.
+
+    :param data: data that would be dumped as JSON
+    :type data: dict
+    :param output_filename: filename where data would be written
+    :type output_filename: str
+    """
+    try:
+        dump = json.dumps(data)
+    except json.JSONDecodeError as err:
+        print(f"couldn't convert results into json format (error: {err})")
+        raise
+
+    pathlib.Path(output_filename).write_text(dump)
+    print(f"Results written as JSON in '{output_filename}'")
+
+
+def append_suffix_to_filename(filename: str, suffix: str, stem: str) -> str:
+    """
+    Appends a suffix to a given filename, considering a specific stem. If the filename
+    already ends with the given stem, the suffix is inserted before the stem. Otherwise,
+    the suffix and stem are added to the end of the filename.
+
+    :param filename: The original filename to which the suffix and stem will be appended
+    :type filename: str
+    :param suffix: The string to be appended as the suffix
+    :type suffix: str
+    :param stem: The specific stem to check or append after the suffix
+    :type stem: str
+
+    :return: The new filename with the appended suffix and stem
+    :rtype: str
+    """
+    filename = filename[:]  # Make a copy to avoid modifying the original filename
+    if filename.endswith(stem):
+        filename = filename[: -len(stem)] + suffix + stem
+    else:
+        filename += suffix + stem
+
+    return filename

ci/perf_regression/perf_regression.py

@@ -13,13 +13,20 @@ One can also provide a fully custom profile via the issue comment string see: fu
 
 This script is also capable of checking for performance regression based on previous benchmarks results.
 It works by providing a result file containing the baseline values and the results of the last run.
+Alongside this mode, a performance report can be generated to help identify potential regressions.
 """
 
 import argparse
 import enum
+import json
+import math
 import pathlib
+import statistics
 import sys
 import tomllib
+from dataclasses import dataclass
+
+from py_markdown_table.markdown_table import markdown_table
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -32,12 +39,28 @@ parser.add_argument(
     dest="issue_comment",
     help="GitHub issue comment defining the regression benchmark profile to use",
 )
+parser.add_argument(
+    "--results-file",
+    dest="results_file",
+    help="Path to the results file containing the baseline and last run results",
+)
+parser.add_argument(
+    "--generate-report",
+    dest="generate_report",
+    action="store_true",
+    default=False,
+    help="Generate markdown report of the regression check",
+)
 
 COMMENT_IDENTIFIER = "/bench"
 
+SECONDS_IN_NANO = 1e9
+MILLISECONDS_IN_NANO = 1e6
+MICROSECONDS_IN_NANO = 1e3
+
 CWD = pathlib.Path(__file__).parent
-REPO_ROOT = CWD.parent
+REPO_ROOT = CWD.parent.parent
-PROFILE_DEFINITION_PATH = CWD.joinpath("regression.toml")
+PROFILE_DEFINITION_PATH = CWD.parent.joinpath("regression.toml")
 BENCH_TARGETS_PATH = REPO_ROOT.joinpath("tfhe-benchmark/Cargo.toml")
 # Files generated after parsing an issue comment
 GENERATED_COMMANDS_PATH = CWD.joinpath("perf_regression_generated_commands.json")
@@ -305,6 +328,8 @@ class ProfileDefinition:
             case TfheBackend.Hpu:
                 features.extend(["hpu", "hpu-v80"])
 
+        features.append("pbs-stats")
+
         return features
 
     def generate_cargo_commands(self):
@@ -415,12 +440,312 @@ def write_backend_config_to_file(backend, profile):
     :return:
     """
     for filepart, content in [("backend", backend), ("profile", profile)]:
-        pathlib.Path(f"ci/perf_regression_slab_{filepart}_config.txt").write_text(
+        CWD.joinpath(f"perf_regression_slab_{filepart}_config.txt").write_text(
             f"{content}\n"
         )
 
 
-# TODO Perform regression computing by providing a file containing results from database that would be parsed
+def write_regression_config_to_file(tfhe_rs_backend, regression_profile):
+    """
+    Write tfhe-rs backend and regression configuration to different files to ease parsing.
+
+    :param backend:
+    :param profile:
+    :return:
+    """
+    for filepart, content in [
+        ("tfhe_rs_backend", tfhe_rs_backend),
+        ("selected_profile", regression_profile),
+    ]:
+        CWD.joinpath(f"perf_regression_{filepart}_config.txt").write_text(
+            f"{content}\n"
+        )
+
+
+# Scale factor to improve the signal/noise ratio in anomaly detection.
+MAJOR_CHANGE_SCALE_FACTOR = 4
+MINOR_CHANGE_SCALE_FACTOR = 2
+REGRESSION_REPORT_FILE = CWD.joinpath("regression_report.md")
+
+
+class PerfChange(enum.StrEnum):
+    NoChange = "no changes"
+    MinorImprovement = "minor improvement"
+    MajorImprovement = "improvement"
+    MinorRegression = "minor regression"
+    MajorRegression = "regression"
+
+    def get_emoji(self):
+        match self:
+            case PerfChange.NoChange:
+                return ":heavy_minus_sign:"
+            case PerfChange.MinorImprovement:
+                return ":white_check_mark:"
+            case PerfChange.MajorImprovement:
+                return ":heavy_check_mark:"
+            case PerfChange.MinorRegression:
+                return ":warning:"
+            case PerfChange.MajorRegression:
+                return ":bangbang:"
+
+
+@dataclass
+class OperationPerformance:
+    name: str
+    baseline_mean: float
+    baseline_stdev: float
+    head_branch_value: float
+    change_percentage: float
+    change_type: PerfChange = PerfChange.NoChange
+
+    def __init__(self, name: str, baseline_data: list[float], head_branch_value: float):
+        self.name = name
+        self.baseline_mean = round(statistics.mean(baseline_data), 2)
+        self.baseline_stdev = round(statistics.stdev(baseline_data), 2)
+        self.head_branch_value = head_branch_value
+
+    def compute_change(self):
+        self.change_percentage = round(
+            (self.head_branch_value - self.baseline_mean) / self.baseline_mean * 100, 2
+        )
+
+        major_threshold = MAJOR_CHANGE_SCALE_FACTOR * self.baseline_stdev
+        minor_threshold = MINOR_CHANGE_SCALE_FACTOR * self.baseline_stdev
+
+        if self.head_branch_value > self.baseline_mean + major_threshold:
+            self.change_type = PerfChange.MajorRegression
+        elif self.head_branch_value > self.baseline_mean + minor_threshold:
+            self.change_type = PerfChange.MinorRegression
+        elif (self.head_branch_value < self.baseline_mean + minor_threshold) and (
+            self.head_branch_value > self.baseline_mean - minor_threshold
+        ):
+            # Between +/- MINOR_CHANGE_SCALE_FACTOR * Std_dev we consider there is no change
+            self.change_type = PerfChange.NoChange
+        elif self.head_branch_value < self.baseline_mean - major_threshold:
+            self.change_type = PerfChange.MajorImprovement
+        elif self.head_branch_value < self.baseline_mean - minor_threshold:
+            self.change_type = PerfChange.MinorImprovement
+
+        return self.change_percentage, self.change_type
+
+    def change_percentage_as_str(self):
+        if (
+            self.change_type == PerfChange.MajorImprovement
+            or self.change_type == PerfChange.MinorImprovement
+            or (
+                self.change_type == PerfChange.NoChange
+                and (self.head_branch_value < self.baseline_mean)
+            )
+        ):
+            sign = ""
+        else:
+            # Minus sign is already embedded in the float value.
+            sign = "+"
+
+        return f"{sign}{self.change_percentage}%"
+
+
+def convert_value_to_readable_text(value_nanos: float, max_digits=3):
+    """
+    Convert timing in nanoseconds to the highest unit usable.
+
+    :param value_nanos: timing value
+    :param max_digits: number of digits to keep in the final representation of the value
+
+    :return: human-readable value with unit as :class:`str`
+    """
+    if value_nanos > SECONDS_IN_NANO:
+        converted_parts = (value_nanos / SECONDS_IN_NANO), "s"
+    elif value_nanos > MILLISECONDS_IN_NANO:
+        converted_parts = (value_nanos / MILLISECONDS_IN_NANO), "ms"
+    elif value_nanos > MICROSECONDS_IN_NANO:
+        converted_parts = (value_nanos / MICROSECONDS_IN_NANO), "us"
+    else:
+        converted_parts = value_nanos, "ns"
+
+    power_of_10 = math.floor(math.log10(converted_parts[0]))
+    rounding_digit = max_digits - (power_of_10 + 1)
+    if converted_parts[0] >= 100.0:
+        rounding_digit = None
+
+    return f"{round(converted_parts[0], rounding_digit)} {converted_parts[1]}"
+
+
+def check_performance_changes(results_file: pathlib.Path):
+    """
+    Check if any operation has regressed compared to the base branch.
+
+    Results file must be in JSON format with the following structure:
+
+    ```json
+    {
+      "backend": "<tfhe-rs_backend>",
+      "profile": "<regression_profile>",
+      "operation": [
+        {
+          "name": "<operation_name>",
+          "bit_size": <int>,
+          "params": "<parameters_alias>",
+
+          "results": {
+            "base": {
+              "name": "<base_branche_name>",
+              "data": [
+                <float>,
+                <float>,
+                ...,
+              ]
+            },
+            "head": {
+              "name": "<dev_branch_name>",
+              "value": <float>
+            }
+          }
+        }
+      ]
+    }
+    ```
+
+    :param results_file: path to the result file
+    :type results_file: pathlib.Path
+
+    :return: :class:`list` of :class:`OperationPerformance`
+    """
+    changes = []
+
+    results = json.loads(results_file.read_text())
+    for operation in results["operations"]:
+        op_name = operation["name"]
+        try:
+            baseline_data = operation["results"]["base"]["data"]
+        except KeyError:
+            raise KeyError(
+                f"no base branch data found in results file for '{op_name}' operation"
+            )
+
+        try:
+            head_branch_value = operation["results"]["head"]["value"]
+        except KeyError:
+            raise KeyError(
+                f"no head branch value found in results file for '{op_name}' operation"
+            )
+
+        op_perf = OperationPerformance(op_name, baseline_data, head_branch_value)
+        op_perf.compute_change()
+        changes.append(op_perf)
+
+    return changes, results["backend"], results["profile"]
+
+
+OPERATION_HEADER = "Operation"
+CURRENT_VALUE_HEADER = "Current (ms)"
+BASELINE_VALUE_HEADER = "Baseline (ms)"
+BASELINE_STDDEV_HEADER = "Baseline Stddev (ms)"
+CHANGE_HEADER = "Change (%)"
+STATUS_HEADER = "Status"
+
+
+def generate_regression_report(
+    ops_performances: list[OperationPerformance], backend: str, profile: str
+):
+    """
+    Generate a regression report in Markdown format and write it to a specified file.
+
+    This function analyzes performance data for various operations and generates a Markdown
+    formatted report summarizing the performance. It highlights any major regressions by
+    providing detailed data about the affected operations and overall benchmark results.
+    Additionally, it includes configuration details, such as the backend and regression profile
+    used in the analysis.
+
+    :param ops_performances: A list of performance data for operations to be analyzed
+        and summarized.
+    :type ops_performances: list[OperationPerformance]
+    :param backend: The backend being used in the performance analysis.
+    :type backend: str
+    :param profile: The profile identifying the regression analysis configuration.
+    :type profile: str
+    """
+    full_data = [
+        {
+            OPERATION_HEADER: op.name,
+            CURRENT_VALUE_HEADER: convert_value_to_readable_text(op.head_branch_value),
+            BASELINE_VALUE_HEADER: convert_value_to_readable_text(op.baseline_mean),
+            BASELINE_STDDEV_HEADER: convert_value_to_readable_text(op.baseline_stdev),
+            CHANGE_HEADER: op.change_percentage_as_str(),
+            STATUS_HEADER: " ".join((op.change_type.get_emoji(), op.change_type.value)),
+        }
+        for op in ops_performances
+    ]
+    full_array_markdown = (
+        markdown_table(full_data)
+        .set_params(row_sep="markdown", quote=False)
+        .get_markdown()
+    )
+
+    regression_data = []
+    for op in ops_performances:
+        if op.change_type != PerfChange.MajorRegression:
+            continue
+
+        regression_data.append(
+            {
+                OPERATION_HEADER: op.name,
+                CURRENT_VALUE_HEADER: convert_value_to_readable_text(
+                    op.head_branch_value
+                ),
+                BASELINE_VALUE_HEADER: convert_value_to_readable_text(op.baseline_mean),
+                CHANGE_HEADER: op.change_percentage_as_str(),
+            }
+        )
+
+    comment_body = []
+
+    if regression_data:
+        regression_array_markdown = (
+            markdown_table(regression_data)
+            .set_params(row_sep="markdown", quote=False)
+            .get_markdown()
+        )
+
+        regression_details = [
+            "> [!CAUTION]",
+            "> Performances for some operations have regressed compared to the base branch.",
+            "",  # Add a newline since tables cannot be rendered in markdown note.
+            regression_array_markdown,
+            "",  # Add a newline to avoid rendering the next line of text into the array.
+        ]
+        comment_body.extend(regression_details)
+    else:
+        comment_body.append("No performance regression detected. :tada:")
+
+    comment_body.append(
+        "\n".join(
+            [
+                "Configuration",
+                f"* backend: `{backend}`",
+                f"* regression-profile: `{profile}`",
+                "",
+            ]
+        )
+    )
+
+    all_results_details = [
+        "<details>",
+        "<summary><strong>View All Benchmarks</strong></summary>",
+        "",
+        full_array_markdown,
+        "</details>",
+    ]
+    comment_body.extend(all_results_details)
+
+    formatted_text = "\n".join(comment_body)
+
+    try:
+        REGRESSION_REPORT_FILE.write_text(formatted_text)
+    except Exception as err:
+        print(f"failed to write regression report (error: {err})")
+        raise
+
 
 if __name__ == "__main__":
     args = parser.parse_args()
@@ -449,8 +774,27 @@ if __name__ == "__main__":
             write_backend_config_to_file(
                 definition.slab_backend, definition.slab_profile
             )
+            write_regression_config_to_file(
+                definition.backend, definition.regression_profile
+            )
         except Exception as err:
             print(f"failed to write commands/env to file (error:{err})")
             sys.exit(3)
     elif args.command == "check_regression":
-        pass
+        results_file = args.results_file
+        if not results_file:
+            print(
+                f"cannot run `{args.command}` command: please specify the results file path with `--results-file` argument"
+            )
+            sys.exit(1)
+
+        results_file_path = pathlib.Path(results_file)
+        perf_changes, backend, profile = check_performance_changes(results_file_path)
+
+        if args.generate_report:
+            try:
+                generate_regression_report(perf_changes, backend, profile)
+            except Exception:
+                sys.exit(4)
+
+# TODO Add unittests primarly to check if commands and env generated are correct.

ci/perf_regression/requirements.txt

@@ -0,0 +1 @@
+py-markdown-table==1.3.0

ci/regression.toml

@@ -42,20 +42,25 @@
 #   See ci/slab.toml file to have the list of all supported profiles.
 
 [gpu.default]
-target.integer-bench = ["mul", "div"]
+target.integer = ["mul"]
-target.hlapi-dex = ["dex_swap"]
+target.hlapi-dex = ["dex::swap_claim::no_cmux"]
 slab.backend = "hyperstack"
 slab.profile = "single-h100"
 env.fast_bench = "TRUE"
+env.bench_param_type = "MULTI_BIT"
 
 [gpu.multi-h100]
-target.integer-bench = ["mul", "div"]
+target.integer = ["mul", "div"]
 target.hlapi-dex = ["dex_swap"]
 slab.backend = "hyperstack"
 slab.profile = "multi-h100"
+env.fast_bench = "TRUE"
+env.bench_param_type = "MULTI_BIT"
 
 [cpu.default]
-target.integer-bench = ["add_parallelized", "mul_parallelized", "div_parallelized"]
+target.integer = ["add_parallelized"]
+target.hlapi-dex = ["dex::swap_claim::no_cmux"]
+target.hlapi-erc20 = ["transfer::whitepaper"]
 slab.backend = "aws"
 slab.profile = "bench"
 env.fast_bench = "TRUE"

ci/slab.toml

@@ -18,7 +18,7 @@ user = "ubuntu"
 
 [backend.aws.bench]
 region = "eu-west-1"
-image_id = "ami-0b8fd02b263ebdcf4"
+image_id = "ami-06e3a9d9b57d38676"
 instance_type = "hpc7a.96xlarge"
 user = "ubuntu"
 

tfhe-benchmark/Cargo.toml

@@ -43,25 +43,25 @@ pbs-stats = ["tfhe/pbs-stats"]
 zk-pok = ["tfhe/zk-pok"]
 
 [[bench]]
-name = "boolean-bench"
+name = "boolean"
 path = "benches/boolean/bench.rs"
 harness = false
 required-features = ["boolean", "internal-keycache"]
 
 [[bench]]
-name = "shortint-bench"
+name = "shortint"
 path = "benches/shortint/bench.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]
 
 [[bench]]
-name = "oprf-shortint-bench"
+name = "shortint-oprf"
 path = "benches/shortint/oprf.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]
 
 [[bench]]
-name = "glwe_packing_compression-shortint-bench"
+name = "shortint-glwe_packing_compression"
 path = "benches/shortint/glwe_packing_compression.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]
@@ -91,55 +91,55 @@ harness = false
 required-features = ["integer", "internal-keycache"]
 
 [[bench]]
-name = "glwe_packing_compression-integer-bench"
+name = "integer-glwe_packing_compression"
 path = "benches/integer/glwe_packing_compression.rs"
 harness = false
 required-features = ["integer", "pbs-stats", "internal-keycache"]
 
 [[bench]]
-name = "integer-bench"
+name = "integer"
 path = "benches/integer/bench.rs"
 harness = false
 required-features = ["integer", "pbs-stats", "internal-keycache"]
 
 [[bench]]
-name = "integer-signed-bench"
+name = "integer-signed"
 path = "benches/integer/signed_bench.rs"
 harness = false
 required-features = ["integer", "pbs-stats", "internal-keycache"]
 
 [[bench]]
-name = "zk-pke-bench"
+name = "integer-zk-pke"
 path = "benches/integer/zk_pke.rs"
 harness = false
 required-features = ["integer", "zk-pok", "pbs-stats", "internal-keycache"]
 
 [[bench]]
-name = "ks-bench"
+name = "core_crypto-ks"
 path = "benches/core_crypto/ks_bench.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]
 
 [[bench]]
-name = "pbs-bench"
+name = "core_crypto-pbs"
 path = "benches/core_crypto/pbs_bench.rs"
 harness = false
 required-features = ["boolean", "shortint", "internal-keycache"]
 
 [[bench]]
-name = "ks-pbs-bench"
+name = "core_crypto-ks-pbs"
 path = "benches/core_crypto/ks_pbs_bench.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]
 
 [[bench]]
-name = "modulus_switch_noise_reduction"
+name = "core_crypto-modulus_switch_noise_reduction"
 path = "benches/core_crypto/modulus_switch_noise_reduction.rs"
 harness = false
 required-features = ["shortint"]
 
 [[bench]]
-name = "pbs128-bench"
+name = "core_crypto-pbs128"
 path = "benches/core_crypto/pbs128_bench.rs"
 harness = false
 required-features = ["shortint", "internal-keycache"]