chore(ci): run erc20 benchmarks in github

It also send results to zama's benchmark database.
2026-01-09 14:47:56 -05:00 · 2024-10-08 09:47:19 +02:00
parent 393e144a64
commit 8da31dbbed
3 changed files with 254 additions and 23 deletions
--- a/.github/workflows/benchmark_erc20.yml
+++ b/.github/workflows/benchmark_erc20.yml
@@ -0,0 +1,144 @@
+# Run all ERC20 benchmarks on an AWS instance and return parsed results to Slab CI bot.
+name: ERC20 benchmarks
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Weekly benchmarks will be triggered each Saturday at 5a.m.
+    - cron: '0 5 * * 6'
+
+env:
+  CARGO_TERM_COLOR: always
+  RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
+  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+  RUST_BACKTRACE: "full"
+  RUST_MIN_STACK: "8388608"
+  SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+  SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+  SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+  SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+
+jobs:
+  setup-instance:
+    name: Setup instance (erc20-benchmarks)
+    runs-on: ubuntu-latest
+    outputs:
+      runner-name: ${{ steps.start-instance.outputs.label }}
+    steps:
+      - name: Start instance
+        id: start-instance
+        uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
+        with:
+          mode: start
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          backend: aws
+          profile: bench
+
+  erc20-benchmarks:
+    name: Execute ERC20 benchmarks
+    needs: setup-instance
+    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
+    concurrency:
+      group: ${{ github.workflow }}_${{ github.ref }}
+      cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+    continue-on-error: true
+    timeout-minutes: 720  # 12 hours
+    steps:
+      - name: Checkout tfhe-rs repo with tags
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.FHE_ACTIONS_TOKEN }}
+
+      - name: Get benchmark details
+        run: |
+          {
+            echo "BENCH_DATE=$(date --iso-8601=seconds)";
+            echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})";
+            echo "COMMIT_HASH=$(git describe --tags --dirty)";
+          } >> "${GITHUB_ENV}"
+
+      - name: Set up home
+        # "Install rust" step require root user to have a HOME directory which is not set.
+        run: |
+          echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
+
+      - name: Install rust
+        uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a
+        with:
+          toolchain: nightly
+
+      - name: Checkout Slab repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938
+        with:
+          repository: zama-ai/slab
+          path: slab
+          token: ${{ secrets.FHE_ACTIONS_TOKEN }}
+
+      - name: Run benchmarks
+        run: |
+          make bench_hlapi_erc20
+
+      - name: Parse results
+        run: |
+          python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \
+          --database tfhe_rs \
+          --hardware "hpc7a.96xlarge" \
+          --project-version "${{ env.COMMIT_HASH }}" \
+          --branch ${{ github.ref_name }} \
+          --commit-date "${{ env.COMMIT_DATE }}" \
+          --bench-date "${{ env.BENCH_DATE }}" \
+          --walk-subdirs \
+          --name-suffix avx512
+
+      - name: Parse PBS counts
+        run: |
+          python3 ./ci/benchmark_parser.py tfhe/erc20_pbs_count.csv ${{ env.RESULTS_FILENAME }} \
+          --key-sizes \
+          --append-results
+
+      - name: Upload parsed results artifact
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+        with:
+          name: ${{ github.sha }}_erc20
+          path: ${{ env.RESULTS_FILENAME }}
+
+      - name: Send data to Slab
+        shell: bash
+        run: |
+          python3 slab/scripts/data_sender.py ${{ env.RESULTS_FILENAME }} "${{ secrets.JOB_SECRET }}" \
+          --slab-url "${{ secrets.SLAB_URL }}"
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "ERC20 benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+
+  teardown-instance:
+    name: Teardown instance (erc20-benchmarks)
+    if: ${{ always() && needs.setup-instance.result != 'skipped' }}
+    needs: [ setup-instance, erc20-benchmarks ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Stop instance
+        id: stop-instance
+        uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
+        with:
+          mode: stop
+          github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
+          slab-url: ${{ secrets.SLAB_BASE_URL }}
+          job-secret: ${{ secrets.JOB_SECRET }}
+          label: ${{ needs.setup-instance.outputs.runner-name }}
+
+      - name: Slack Notification
+        if: ${{ failure() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "Instance teardown (erc20-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
--- a/6
+++ b/6
@@ -1149,6 +1149,12 @@ bench_web_js_api_parallel_firefox_ci: setup_venv
 	nvm use $(NODE_VERSION) && \
 	$(MAKE) bench_web_js_api_parallel_firefox

+.PHONY: bench_hlapi_erc20 # Run benchmarks for ECR20 operations
+bench_hlapi_erc20: install_rs_check_toolchain
+	RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) bench \
+	--bench hlapi-erc20 \
+	--features=$(TARGET_ARCH_FEATURE),integer,internal-keycache,pbs-stats,nightly-avx512 -p $(TFHE_SPEC) --
+
 #
 # Utility tools
 #
--- a/tfhe/benches/high_level_api/erc20.rs
+++ b/tfhe/benches/high_level_api/erc20.rs
@@ -1,9 +1,14 @@
+#[path = "../utilities.rs"]
+mod utilities;
+
+use crate::utilities::{write_to_json, OperatorType};
 use criterion::measurement::WallTime;
 use criterion::{BenchmarkGroup, Criterion, Throughput};
 use rand::prelude::*;
 use rand::thread_rng;
 use rayon::prelude::*;
 use std::ops::{Add, Mul, Sub};
+use tfhe::keycache::NamedParam;
 use tfhe::prelude::*;
 use tfhe::shortint::parameters::*;
 use tfhe::{set_server_key, ClientKey, CompressedServerKey, ConfigBuilder, FheBool, FheUint64};
@@ -103,31 +108,69 @@ where
 }

 #[cfg(feature = "pbs-stats")]
-fn print_transfer_pbs_counts<FheType, F>(
-    client_key: &ClientKey,
-    type_name: &str,
-    fn_name: &str,
-    transfer_func: F,
-) where
-    FheType: FheEncrypt<u64, ClientKey>,
-    F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType),
-{
-    let mut rng = thread_rng();
+mod pbs_stats {
+    use super::*;
+    use std::fs::{File, OpenOptions};
+    use std::io::Write;
+    use std::path::Path;

-    let from_amount = FheType::encrypt(rng.gen::<u64>(), client_key);
-    let to_amount = FheType::encrypt(rng.gen::<u64>(), client_key);
-    let amount = FheType::encrypt(rng.gen::<u64>(), client_key);
+    fn write_result(file: &mut File, name: &str, value: usize) {
+        let line = format!("{name},{value}\n");
+        let error_message = format!("cannot write {name} result into file");
+        file.write_all(line.as_bytes()).expect(&error_message);
+    }

-    tfhe::reset_pbs_count();
-    let (_, _) = transfer_func(&from_amount, &to_amount, &amount);
-    let count = tfhe::get_pbs_count();
+    pub fn print_transfer_pbs_counts<FheType, F>(
+        client_key: &ClientKey,
+        type_name: &str,
+        fn_name: &str,
+        transfer_func: F,
+    ) where
+        FheType: FheEncrypt<u64, ClientKey>,
+        F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType),
+    {
+        let mut rng = thread_rng();

-    println!("ERC20 transfer/{fn_name}::{type_name}: {count} PBS");
+        let from_amount = FheType::encrypt(rng.gen::<u64>(), client_key);
+        let to_amount = FheType::encrypt(rng.gen::<u64>(), client_key);
+        let amount = FheType::encrypt(rng.gen::<u64>(), client_key);
+
+        tfhe::reset_pbs_count();
+        let (_, _) = transfer_func(&from_amount, &to_amount, &amount);
+        let count = tfhe::get_pbs_count();
+
+        println!("ERC20 transfer/{fn_name}::{type_name}: {count} PBS");
+
+        let params = client_key.computation_parameters();
+        let test_name = format!("hlapi::erc20::pbs_count::{fn_name}::{type_name}");
+
+        let results_file = Path::new("erc20_pbs_count.csv");
+        if !results_file.exists() {
+            File::create(results_file).expect("create results file failed");
+        }
+        let mut file = OpenOptions::new()
+            .append(true)
+            .open(results_file)
+            .expect("cannot open results file");
+
+        write_result(&mut file, &test_name, count as usize);
+
+        write_to_json::<u64, _>(
+            &test_name,
+            params,
+            params.name(),
+            "pbs-count",
+            &OperatorType::Atomic,
+            0,
+            vec![],
+        );
+    }
 }

 fn bench_transfer_latency<FheType, F>(
    c: &mut BenchmarkGroup<'_, WallTime>,
    client_key: &ClientKey,
+    bench_name: &str,
    type_name: &str,
    fn_name: &str,
    transfer_func: F,
@@ -135,8 +178,8 @@ fn bench_transfer_latency<FheType, F>(
    FheType: FheEncrypt<u64, ClientKey>,
    F: for<'a> Fn(&'a FheType, &'a FheType, &'a FheType) -> (FheType, FheType),
 {
-    let id_name = format!("{fn_name}::{type_name}");
-    c.bench_function(&id_name, |b| {
+    let bench_id = format!("{bench_name}::{fn_name}::{type_name}");
+    c.bench_function(&bench_id, |b| {
        let mut rng = thread_rng();

        let from_amount = FheType::encrypt(rng.gen::<u64>(), client_key);
@@ -147,11 +190,24 @@ fn bench_transfer_latency<FheType, F>(
            let (_, _) = transfer_func(&from_amount, &to_amount, &amount);
        })
    });
+
+    let params = client_key.computation_parameters();
+
+    write_to_json::<u64, _>(
+        &bench_id,
+        params,
+        params.name(),
+        "erc20-transfer",
+        &OperatorType::Atomic,
+        64,
+        vec![],
+    );
 }

 fn bench_transfer_throughput<FheType, F>(
    group: &mut BenchmarkGroup<'_, WallTime>,
    client_key: &ClientKey,
+    bench_name: &str,
    type_name: &str,
    fn_name: &str,
    transfer_func: F,
@@ -163,8 +219,8 @@ fn bench_transfer_throughput<FheType, F>(

    for num_elems in [10, 100, 500] {
        group.throughput(Throughput::Elements(num_elems));
-        let id_name = format!("{fn_name}::{type_name}::{num_elems}");
-        group.bench_with_input(id_name, &num_elems, |b, &num_elems| {
+        let bench_id = format!("{bench_name}::{fn_name}::{type_name}::{num_elems}_elems");
+        group.bench_with_input(&bench_id, &num_elems, |b, &num_elems| {
            let from_amounts = (0..num_elems)
                .map(|_| FheType::encrypt(rng.gen::<u64>(), client_key))
                .collect::<Vec<_>>();
@@ -184,9 +240,24 @@ fn bench_transfer_throughput<FheType, F>(
                    })
            })
        });
+
+        let params = client_key.computation_parameters();
+
+        write_to_json::<u64, _>(
+            &bench_id,
+            params,
+            params.name(),
+            "erc20-transfer",
+            &OperatorType::Atomic,
+            64,
+            vec![],
+        );
    }
 }

+#[cfg(feature = "pbs-stats")]
+use pbs_stats::print_transfer_pbs_counts;
+
 fn main() {
    #[cfg(not(feature = "gpu"))]
    let params = PARAM_MESSAGE_2_CARRY_2_KS_PBS_TUNIFORM_2M64;
@@ -230,10 +301,12 @@ fn main() {

    // FheUint64 latency
    {
-        let mut group = c.benchmark_group("ERC20 latency");
+        let bench_name = "hlapi::erc20::transfer_latency";
+        let mut group = c.benchmark_group(bench_name);
        bench_transfer_latency(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "whitepaper",
            transfer_whitepaper::<FheUint64>,
@@ -241,6 +314,7 @@ fn main() {
        bench_transfer_latency(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "no_cmux",
            transfer_no_cmux::<FheUint64>,
@@ -248,6 +322,7 @@ fn main() {
        bench_transfer_latency(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "overflow",
            transfer_overflow::<FheUint64>,
@@ -255,6 +330,7 @@ fn main() {
        bench_transfer_latency(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "safe",
            transfer_safe::<FheUint64>,
@@ -265,10 +341,12 @@ fn main() {

    // FheUint64 Throughput
    {
-        let mut group = c.benchmark_group("ERC20 throughput");
+        let bench_name = "hlapi::erc20::transfer_throughput";
+        let mut group = c.benchmark_group(bench_name);
        bench_transfer_throughput(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "whitepaper",
            transfer_whitepaper::<FheUint64>,
@@ -276,6 +354,7 @@ fn main() {
        bench_transfer_throughput(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "no_cmux",
            transfer_no_cmux::<FheUint64>,
@@ -283,6 +362,7 @@ fn main() {
        bench_transfer_throughput(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "overflow",
            transfer_overflow::<FheUint64>,
@@ -290,6 +370,7 @@ fn main() {
        bench_transfer_throughput(
            &mut group,
            &cks,
+            bench_name,
            "FheUint64",
            "safe",
            transfer_safe::<FheUint64>,