chore: benchmark halo2-eth-membership (#521)

* feat: benchmarking halo2-eth-membership

* feat: adding a benchmark plot

* feat: enhance circuit config

* chore: add changeset

* refactor: ignore benchmark results files

* refactor: rename config extension to .cfg and update README with benchmark

* chore: add changeset

* Delete .changeset/short-cows-shake.md

.changeset/afraid-carrots-exist.md

@@ -0,0 +1,5 @@
+---
+"@anonklub/halo2-eth-membership": minor
+---
+
+Adding benchmarking report & changing config to be using k=14

.gitignore

@@ -46,3 +46,7 @@ prove_test_inputs.json
 verify_test_inputs.json
 
 test_inputs.json
+
+# Benchmark results
+pkgs/halo2-eth-membership/configs/data
+pkgs/halo2-eth-membership/configs/results

Cargo.lock

@@ -2588,6 +2588,7 @@ name = "halo2-eth-membership"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "ark-std 0.4.0",
  "bincode",
  "ethers",
  "getrandom 0.2.15",

README.md

@@ -114,4 +114,73 @@ Check available scripts with `pnpm run`.\
 Especially, to start the ui or the query-api: `pnpm start.ui` or `pnpm start.query-api`.\
 Don't bother running build tasks explicitly beforehand: [turbo](https://turbo.build/repo/docs) takes care of topological dependencies between tasks.
 
+## Benchmarks
+
+### Overview
+
+This benchmarking process evaluates the relationship between the parameter `k` and the performance metrics such as a proof generation time, proof size, and verification time. The results are stored in a CSV file and visualized using plots.
+
+### Setup
+
+Before running the benchmarks, ensure that your Rust environment is set up correctly and that the appropriate target is configured.
+
+#### Configuring the Rust Target
+
+This benchmarking process is intended to run on a native Linux target, not on WebAssembly (WASM). The project uses a specific Rust toolchain version.
+
+1. Set Up the Rust Toolchain:
+   Ensure to use the same nightly Rust version in the `rust-toolchain`.
+
+```rs
+[toolchain]
+channel = "nightly-2024-07-25"
+```
+
+2. Check the Installed Targets:
+   You can check which targets are installed in your Rust toolchain with:
+
+```bash
+rustup target list --installed
+```
+
+3. Add the Required Target:
+   If the `x86_64-unknown-linux-gnu` target is not installed, you can add it with:
+
+```bash
+rustup target add x86_64-unknown-linux-gnu
+```
+
+4. Run the Benchmark:
+   Ensure that the benchmark is run with the correct target by using the following command:
+
+```bash
+cargo test --target x86_64-unknown-linux-gnu --features "bench"
+```
+
+### Results
+
+The benchmark results were obtained on `Lenovo Legion 5` running Linux (12 CPU cores, 62 GB RAM) -- no GPU was used.
+
+| k  | numAdvice | numLookupAdvice | numInstance | numLookupBits | numVirtualInstance | proof_time | proof_size | verify_time |
+| -- | --------- | --------------- | ----------- | ------------- | ------------------ | ---------- | ---------- | ----------- |
+| 19 | 1         | 1               | 1           | 18            | 1                  | 176.9s     | 992        | 2.3s        |
+| 18 | 2         | 1               | 1           | 17            | 1                  | 171.1s     | 1504       | 7.6s        |
+| 17 | 4         | 1               | 1           | 16            | 1                  | 71.7s      | 2080       | 639.7ms     |
+| 16 | 8         | 2               | 1           | 15            | 1                  | 59.3s      | 3584       | 365.1ms     |
+| 15 | 17        | 3               | 1           | 14            | 1                  | 51.2s      | 6592       | 267.6ms     |
+| 14 | 34        | 6               | 1           | 13            | 1                  | 51.6s      | 12736      | 283.8ms     |
+| 13 | 68        | 12              | 1           | 12            | 1                  | 52.5s      | 25024      | 411.5ms     |
+| 12 | 139       | 24              | 1           | 11            | 1                  | 58.3s      | 50528      | 761.7ms     |
+| 11 | 291       | 53              | 1           | 10            | 1                  | 72.4s      | 106304     | 1.5s        |
+
+> Note: those benchmark config parameters have been selected based on `halo2-lib` benchmark params [github.com/axiom-crypto/halo2-lib](https://github.com/axiom-crypto/halo2-lib?tab=readme-ov-file#secp256k1-ecdsa)
+
+The benchmark results are visualized in the plot below:
+
+![Benchmark Plot](pkgs/halo2-eth-membership/configs//benchmark_plot.png)
+
+### Results using `criterion.rs`
+
+TODO
+
 ## [Contribute](https://github.com/anonklub/anonklub/contribute)

pkgs/halo2-eth-membership/Cargo.toml

@@ -11,6 +11,7 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 anyhow = "1.0.81"
+ark-std = { version = "0.4.0", features = ["print-trace"] }
 bincode = "1.3.3"
 ethers = "2.0.11"
 getrandom = { version = "0.2", features = ["js"] }
@@ -44,3 +45,4 @@ wasm-bindgen-test = "0.3"
 [features]
 default = ["rayon"]
 rayon = ["halo2-wasm/rayon"]
+bench = ["rayon"]

pkgs/halo2-eth-membership/build.rs

@@ -8,7 +8,7 @@ fn main() {
     let dest_path = Path::new(&out_dir).join("eth_membership_config.rs");
 
     let config_data =
-        fs::read_to_string("configs/eth_membership.config").expect("Unable to read config file");
+        fs::read_to_string("configs/eth_membership.cfg").expect("Unable to read config file");
 
     fs::write(
         &dest_path,

pkgs/halo2-eth-membership/configs/benchmark.cfg

@@ -0,0 +1,9 @@
+{"k":19,"numAdvice":1,"numLookupAdvice":1,"numInstance":1,"numLookupBits":18,"numVirtualInstance":1}
+{"k":18,"numAdvice":2,"numLookupAdvice":1,"numInstance":1,"numLookupBits":17,"numVirtualInstance":1}
+{"k":17,"numAdvice":4,"numLookupAdvice":1,"numInstance":1,"numLookupBits":16,"numVirtualInstance":1}
+{"k":16,"numAdvice":8,"numLookupAdvice":2,"numInstance":1,"numLookupBits":15,"numVirtualInstance":1}
+{"k":15,"numAdvice":17,"numLookupAdvice":3,"numInstance":1,"numLookupBits":14,"numVirtualInstance":1}
+{"k":14,"numAdvice":34,"numLookupAdvice":6,"numInstance":1,"numLookupBits":13,"numVirtualInstance":1}
+{"k":13,"numAdvice":68,"numLookupAdvice":12,"numInstance":1,"numLookupBits":12,"numVirtualInstance":1}
+{"k":12,"numAdvice":139,"numLookupAdvice":24,"numInstance":1,"numLookupBits":11,"numVirtualInstance":1}
+{"k":11,"numAdvice":291,"numLookupAdvice":53,"numInstance":1,"numLookupBits":10,"numVirtualInstance":1}

pkgs/halo2-eth-membership/configs/eth_membership.cfg

@@ -1,8 +1,8 @@
 {
-  "k": 15,
+  "k": 14,
   "numAdvice": 34,
-  "numLookupAdvice": 3,
+  "numLookupAdvice": 6,
   "numInstance": 1,
-  "numLookupBits": 14,
+  "numLookupBits": 13,
   "numVirtualInstance": 1
 }

pkgs/halo2-eth-membership/src/eth_membership.rs

@@ -391,7 +391,7 @@ mod mock_tests {
     use halo2_ecdsa::utils::verify::verify_efficient_ecdsa;
     use halo2_wasm::{halo2lib::ecc::Secp256k1Affine, CircuitConfig, Halo2Wasm};
     use halo2_wasm_ext::consts::F;
-    use halo2_wasm_ext::ext::Halo2WasmExt;
+    use halo2_wasm_ext::ext::{CircuitConfigExt, Halo2WasmExt};
     use halo2_wasm_ext::params::serialize_params_to_bytes;
     use halo2_wasm_ext::utils::ct_option_ok_or;
     use num_bigint::BigUint;
@@ -399,7 +399,7 @@ mod mock_tests {
     use rand_core::OsRng;
     use serde::{Deserialize, Serialize};
     use std::collections::HashMap;
-    use std::io::Read;
+    use std::io::{BufRead, Read, Write};
     use std::{fs::File, time::Instant};
 
     use crate::utils::consts::{E, RATE_POSEIDON, R_F_POSEIDON, R_P_POSEIDON, T_POSEIDON};
@@ -511,7 +511,7 @@ mod mock_tests {
 
     #[test]
     fn test_mock_inputs_eth_membership_mock_prover() -> Result<()> {
-        let path = "configs/eth_membership.config";
+        let path = "configs/eth_membership.cfg";
         let circuit_params: CircuitConfig = serde_json::from_reader(
             File::open(path)
                 .map_err(|e| anyhow!(e))
@@ -554,7 +554,7 @@ mod mock_tests {
 
     #[test]
     fn test_mock_inputs_eth_membership_real_prover_verifier() -> Result<()> {
-        let path = "configs/eth_membership.config";
+        let path = "configs/eth_membership.cfg";
         let circuit_params: CircuitConfig = serde_json::from_reader(
             File::open(path)
                 .map_err(|e| anyhow!(e))
@@ -648,9 +648,179 @@ mod mock_tests {
         Ok(())
     }
 
+    #[cfg(feature = "bench")]
+    #[test]
+    fn bench_test_mock_inputs_eth_membership_real_prover_verifier() -> Result<()> {
+        use ark_std::{end_timer, start_timer};
+
+        let mut folder = std::path::PathBuf::new();
+        folder.push("configs/benchmark.cfg");
+
+        let bench_params_file = std::fs::File::open(folder.as_path()).unwrap();
+        folder.pop();
+
+        folder.push("results");
+        std::fs::create_dir_all(&folder).expect("Failed to create directories");
+
+        folder.push("eff_ecdsa_bench.csv");
+        let mut fs_results =
+            std::fs::File::create(folder.as_path()).expect("Failed to create file");
+        folder.pop();
+        folder.pop();
+
+        writeln!(fs_results, "k,numAdvice,numLookupAdvice,numInstance,numLookupBits,numVirtualInstance,proof_time,proof_size,verify_time")?;
+        folder.push("data");
+
+        if !folder.is_dir() {
+            std::fs::create_dir(folder.as_path())?;
+        }
+
+        let bench_params_reader = std::io::BufReader::new(bench_params_file);
+
+        for line in bench_params_reader.lines() {
+            let line = line.expect("Failed to read a line");
+            let line_str = line.as_str();
+
+            let bench_params: CircuitConfig = serde_json::from_str(line_str)
+                .map_err(|e| anyhow!(e))
+                .with_context(|| format!("Failed to read the circuit config file"))?;
+
+            let bench_params_ext: CircuitConfigExt = serde_json::from_str(line_str)
+                .map_err(|e| anyhow!(e))
+                .with_context(|| format!("Failed to read the circuit config Ext file"))?;
+
+            println!(
+                "---------------------- degree = {} ------------------------------",
+                bench_params_ext.k
+            );
+
+            let params_time = start_timer!(|| "Time elapsed in circuit & params construction");
+
+            let (ecdsa_inputs, test_inputs) = mock_eff_ecdsa_input(PRIV_KEY)?;
+            let merkle_proof = mock_merkle_proof(&test_inputs.address)?;
+            let eth_membership_inputs = EthMembershipInputs::<
+                secp256k1::Fp,
+                secp256k1::Fq,
+                Secp256k1Affine,
+            >::new(ecdsa_inputs, merkle_proof);
+
+            let mut halo2_wasm = Halo2Wasm::new();
+            halo2_wasm.config(bench_params);
+
+            let mut circuit =
+                EthMembershipCircuit::<secp256k1::Fp, secp256k1::Fq, Secp256k1Affine>::new(
+                    &halo2_wasm,
+                    eth_membership_inputs,
+                )?;
+
+            circuit.verify_membership();
+
+            halo2_wasm.set_instances(&circuit.instances, INSTANCE_COL);
+
+            halo2_wasm.assign_instances();
+
+            let params = ParamsKZG::<Bn256>::setup(bench_params_ext.k.try_into().unwrap(), OsRng);
+
+            // Load params
+            halo2_wasm.load_params(&serialize_params_to_bytes(&params));
+
+            end_timer!(params_time);
+
+            // Generate VK
+            let vk_time = start_timer!(|| "Time elapsed in generating vkey");
+            halo2_wasm.gen_vk();
+            end_timer!(vk_time);
+
+            // Generate PK
+            let pk_time = start_timer!(|| "Time elapsed in generating pkey");
+            halo2_wasm.gen_pk();
+            end_timer!(pk_time);
+
+            let start = Instant::now();
+
+            // Time tracking for proof generation
+            let proof_start = Instant::now();
+
+            // Get the public instance inputs
+            let instances = halo2_wasm.get_instance_values_ext(INSTANCE_COL)?;
+
+            // Generate proof
+            let proof_time = start_timer!(|| "Proving time");
+            let proof: Vec<u8> = halo2_wasm.prove_ext(&params);
+            end_timer!(proof_time);
+
+            let proof_duration = proof_start.elapsed();
+            println!(
+                "Eth Membership Proof generation executed in: {:.2?} seconds",
+                proof_duration
+            );
+
+            let proof_size = {
+                folder.push(format!(
+                    "ecdsa_circuit_proof_{}_{}_{}_{}_{}_{}.data",
+                    bench_params_ext.k,
+                    bench_params_ext.num_advice,
+                    bench_params_ext.num_lookup_advice,
+                    bench_params_ext.num_instance,
+                    bench_params_ext.num_lookup_bits,
+                    bench_params_ext.num_virtual_instance
+                ));
+                let mut fd = std::fs::File::create(folder.as_path()).unwrap();
+                folder.pop();
+                fd.write_all(&proof).unwrap();
+                fd.metadata().unwrap().len()
+            };
+
+            // Verify the proof
+            println!("Verifying Proof");
+            let verify_time = start_timer!(|| "Verify time");
+            let is_proof_valid = halo2_wasm.verify_ext(&instances, &proof, params)?;
+            end_timer!(verify_time);
+
+            println!("- Is proof valid? {}", is_proof_valid);
+
+            assert!(is_proof_valid, "The proof is not valid");
+
+            // Verify Eff ECDSA
+            println!("Verifying Eth Membership Proof");
+
+            let is_eff_ecdsa_valid = verify_efficient_ecdsa(
+                test_inputs.msg_hash,
+                test_inputs.r,
+                test_inputs.is_y_odd,
+                &instances,
+            )?;
+
+            println!("- Is Eff ECDSA valid? {}", is_eff_ecdsa_valid);
+
+            assert!(is_eff_ecdsa_valid, "Eff ECDSA is not valid");
+
+            let duration = start.elapsed();
+            let duration_in_minutes = duration.as_secs_f64() / 60.0;
+            println!("Test executed in: {:.2?} seconds", duration);
+            println!("Test executed in: {:.2?} minutes", duration_in_minutes);
+
+            writeln!(
+                fs_results,
+                "{},{},{},{},{},{},{:?},{},{:?}",
+                bench_params_ext.k,
+                bench_params_ext.num_advice,
+                bench_params_ext.num_lookup_advice,
+                bench_params_ext.num_instance,
+                bench_params_ext.num_lookup_bits,
+                bench_params_ext.num_virtual_instance,
+                proof_time.time.elapsed(),
+                proof_size,
+                verify_time.time.elapsed()
+            )?;
+        }
+
+        Ok(())
+    }
+
     #[test]
     fn test_eff_ecdsa_verification() -> Result<()> {
-        let path = "configs/eth_membership.config";
+        let path = "configs/eth_membership.cfg";
         let circuit_params: CircuitConfig = serde_json::from_reader(
             File::open(path)
                 .map_err(|e| anyhow!(e))
@@ -794,7 +964,7 @@ mod real_tests {
         let msg_hash = map_to_vec(&inputs.msg_hash);
         let merkle_proof_bytes_serialized = map_to_vec(&inputs.merkle_proof_bytes_serialized);
 
-        let path = "configs/eth_membership.config";
+        let path = "configs/eth_membership.cfg";
         let circuit_params: CircuitConfig = serde_json::from_reader(
             File::open(path)
                 .map_err(|e| anyhow!(e))

pkgs/halo2-wasm-ext/src/ext.rs

@@ -22,12 +22,24 @@ use halo2_base::{
 };
 use halo2_wasm::Halo2Wasm;
 use itertools::Itertools;
+use serde::{Deserialize, Serialize};
 use snark_verifier_sdk::{
     halo2::{gen_snark_shplonk, PoseidonTranscript, POSEIDON_SPEC},
     NativeLoader,
 };
 use std::io::BufReader;
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CircuitConfigExt {
+    pub k: usize,
+    pub num_advice: usize,
+    pub num_lookup_advice: usize,
+    pub num_instance: usize,
+    pub num_lookup_bits: usize,
+    pub num_virtual_instance: usize,
+}
+
 pub trait Halo2WasmExt {
     fn get_instance_values_ext(&mut self, col: usize) -> Result<Vec<u8>>;
     fn prove_ext(&self, params: &ParamsKZG<E>) -> Vec<u8>;