From 57e5b0dd01d06ff14dafbf2946c59edd1fb74153 Mon Sep 17 00:00:00 2001 From: dante <45801863+alexander-camuto@users.noreply.github.com> Date: Mon, 6 Mar 2023 19:18:41 +0000 Subject: [PATCH] refactor: switch to rawbytes serialization for vk, pk, and srs (#154) --- README.md | 14 ++++++++++++-- src/execute.rs | 25 +++++++++++++++---------- src/pfsys/mod.rs | 38 +++++++++----------------------------- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index c512cfa7..3ed7a5d8 100644 --- a/README.md +++ b/README.md @@ -222,9 +222,13 @@ For instance: ezkl send-proof-evm -S ./mymnemonic.txt -U myethnode.xyz --addr 0xFFFF --proof-path my.snark ``` -#### using pre-generated SRS -Note that you can use pre-generated KZG SRS. These SRS can be converted to a format that is ingestable by the `pse/halo2` prover ezkl uses by leveraging [han0110/halo2-kzg-srs](https://github.com/han0110/halo2-kzg-srs). This repo also contains pre-converted SRS from large projects such as Hermez and the [perpetual powers of tau repo](https://github.com/privacy-scaling-explorations/perpetualpowersoftau). Simply download the pre-converted file locally and point `--params-path` to the file ! +### using pre-generated SRS + +Note that you can use pre-generated KZG SRS. These SRS can be converted to a format that is ingestable by the `pse/halo2` prover ezkl uses by leveraging [han0110/halo2-kzg-srs](https://github.com/han0110/halo2-kzg-srs). This repo also contains pre-converted SRS from large projects such as Hermez and the [perpetual powers of tau repo](https://github.com/privacy-scaling-explorations/perpetualpowersoftau). Simply download the pre-converted file locally and point `--params-path` to the file. + +> Note: Ensure you download the files in raw format. As this will be more performant and is the serialization format `ezkl` assumes. + ### general usage 🔧 @@ -315,6 +319,7 @@ EZKLCONF=/path/to/fullconfig.json ezkl ``` +---------------------- ## benchmarks ⏳ @@ -336,6 +341,8 @@ criterion_group! { targets = runrelu } ``` +---------------------- + ## onnx examples @@ -349,6 +356,9 @@ If you want to add a model to `examples/onnx`, open a PR creating a new folder w TODO: add associated python files in the onnx model directories. +---------------------- + + ## library examples 🔍 Beyond the `.onnx` examples detailed above, we also include examples which directly use some of our rust API; allowing users to code up computational graphs and circuits from scratch in rust without having to go via python. diff --git a/src/execute.rs b/src/execute.rs index 33433c6c..1cff1ff0 100644 --- a/src/execute.rs +++ b/src/execute.rs @@ -9,10 +9,10 @@ use crate::pfsys::evm::single::gen_evm_verifier; use crate::pfsys::evm::{evm_verify, DeploymentCode}; #[cfg(feature = "render")] use crate::pfsys::prepare_model_circuit; -use crate::pfsys::{create_keys, load_params_kzg, load_vk, Snark}; +use crate::pfsys::{create_keys, load_params, load_vk, save_params, Snark}; use crate::pfsys::{ - create_proof_circuit, gen_srs, prepare_data, prepare_model_circuit_and_public_input, - save_params_kzg, save_vk, verify_proof_circuit, + create_proof_circuit, gen_srs, prepare_data, prepare_model_circuit_and_public_input, save_vk, + verify_proof_circuit, }; use halo2_proofs::dev::VerifyFailure; use halo2_proofs::plonk::{Circuit, ProvingKey, VerifyingKey}; @@ -162,7 +162,7 @@ pub async fn run(cli: Cli) -> Result<(), Box> { } Commands::GenSrs { params_path } => { let params = gen_srs::>(cli.args.logrows); - save_params_kzg(¶ms_path, ¶ms)?; + save_params::>(¶ms_path, ¶ms)?; } Commands::Table { model: _ } => { let om = Model::from_ezkl_conf(cli)?; @@ -236,7 +236,8 @@ pub async fn run(cli: Cli) -> Result<(), Box> { let (_, public_inputs) = prepare_model_circuit_and_public_input::(&data, &cli)?; let num_instance = public_inputs.iter().map(|x| x.len()).collect(); - let mut params: ParamsKZG = load_params_kzg(params_path.to_path_buf())?; + let mut params: ParamsKZG = + load_params::>(params_path.to_path_buf())?; if cli.args.logrows < params.k() { params.downsize(cli.args.logrows); } @@ -263,7 +264,7 @@ pub async fn run(cli: Cli) -> Result<(), Box> { deployment_code_path, vk_path, } => { - let params: ParamsKZG = load_params_kzg(params_path)?; + let params: ParamsKZG = load_params::>(params_path)?; let agg_vk = load_vk::, Fr, AggregationCircuit>(vk_path)?; @@ -287,7 +288,8 @@ pub async fn run(cli: Cli) -> Result<(), Box> { let data = prepare_data(data.to_string())?; let (circuit, public_inputs) = prepare_model_circuit_and_public_input(&data, &cli)?; - let mut params: ParamsKZG = load_params_kzg(params_path.to_path_buf())?; + let mut params: ParamsKZG = + load_params::>(params_path.to_path_buf())?; if cli.args.logrows < params.k() { params.downsize(cli.args.logrows); } @@ -339,7 +341,8 @@ pub async fn run(cli: Cli) -> Result<(), Box> { transcript, } => { // the K used for the aggregation circuit - let mut params: ParamsKZG = load_params_kzg(params_path.to_path_buf())?; + let mut params: ParamsKZG = + load_params::>(params_path.to_path_buf())?; if cli.args.logrows < params.k() { params.downsize(cli.args.logrows); } @@ -391,7 +394,8 @@ pub async fn run(cli: Cli) -> Result<(), Box> { params_path, transcript, } => { - let mut params: ParamsKZG = load_params_kzg(params_path)?; + let mut params: ParamsKZG = + load_params::>(params_path)?; if cli.args.logrows < params.k() { params.downsize(cli.args.logrows); } @@ -416,7 +420,8 @@ pub async fn run(cli: Cli) -> Result<(), Box> { params_path, transcript, } => { - let mut params: ParamsKZG = load_params_kzg(params_path)?; + let mut params: ParamsKZG = + load_params::>(params_path)?; if cli.args.logrows < params.k() { params.downsize(cli.args.logrows); } diff --git a/src/pfsys/mod.rs b/src/pfsys/mod.rs index 8aa11bde..5213eb5b 100644 --- a/src/pfsys/mod.rs +++ b/src/pfsys/mod.rs @@ -14,10 +14,8 @@ use halo2_proofs::plonk::{ create_proof, keygen_pk, keygen_vk, verify_proof, Circuit, ProvingKey, VerifyingKey, }; use halo2_proofs::poly::commitment::{CommitmentScheme, Params, ParamsProver, Prover, Verifier}; -use halo2_proofs::poly::kzg::commitment::ParamsKZG; use halo2_proofs::poly::VerificationStrategy; use halo2_proofs::transcript::{EncodedChallenge, TranscriptReadBuffer, TranscriptWriterBuffer}; -use halo2curves::bn256::Bn256; use halo2curves::group::ff::PrimeField; use halo2curves::serde::SerdeObject; use halo2curves::CurveAffine; @@ -421,7 +419,7 @@ where info!("loading verification key from {:?}", path); let f = File::open(path).map_err(Box::::from)?; let mut reader = BufReader::new(f); - VerifyingKey::::read::<_, C>(&mut reader, halo2_proofs::SerdeFormat::Processed) + VerifyingKey::::read::<_, C>(&mut reader, halo2_proofs::SerdeFormat::RawBytes) .map_err(Box::::from) } @@ -437,7 +435,7 @@ where info!("loading proving key from {:?}", path); let f = File::open(path).map_err(Box::::from)?; let mut reader = BufReader::new(f); - ProvingKey::::read::<_, C>(&mut reader, halo2_proofs::SerdeFormat::Processed) + ProvingKey::::read::<_, C>(&mut reader, halo2_proofs::SerdeFormat::RawBytes) .map_err(Box::::from) } @@ -451,15 +449,6 @@ pub fn load_params( Params::<'_, Scheme::Curve>::read(&mut reader).map_err(Box::::from) } -/// Loads the [ParamsKZG] at `path`. -pub fn load_params_kzg(path: PathBuf) -> Result, Box> { - info!("loading params from {:?}", path); - let f = File::open(path).map_err(Box::::from)?; - let mut reader = BufReader::new(f); - ParamsKZG::::read_custom(&mut reader, halo2_proofs::SerdeFormat::Processed) - .map_err(Box::::from) -} - /// Saves a [ProvingKey] to `path`. pub fn save_pk( path: &PathBuf, @@ -472,7 +461,7 @@ where info!("saving proving key 💾"); let f = File::create(path)?; let mut writer = BufWriter::new(f); - vk.write(&mut writer, halo2_proofs::SerdeFormat::Processed)?; + vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?; writer.flush()?; Ok(()) } @@ -489,7 +478,7 @@ where info!("saving verification key 💾"); let f = File::create(path)?; let mut writer = BufWriter::new(f); - vk.write(&mut writer, halo2_proofs::SerdeFormat::Processed)?; + vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?; writer.flush()?; Ok(()) } @@ -507,16 +496,6 @@ pub fn save_params( Ok(()) } -/// Saves [ParamsKZG] to `path`. -pub fn save_params_kzg(path: &PathBuf, params: &'_ ParamsKZG) -> Result<(), io::Error> { - info!("saving parameters 💾"); - let f = File::create(path)?; - let mut writer = BufWriter::new(f); - params.write_custom(&mut writer, halo2_proofs::SerdeFormat::Processed)?; - writer.flush()?; - Ok(()) -} - //////////////////////// #[cfg(test)] @@ -525,13 +504,14 @@ mod tests { use super::*; use halo2_proofs::poly::kzg::commitment::KZGCommitmentScheme; + use halo2curves::bn256::Bn256; use tempfile::Builder; #[tokio::test] async fn test_can_load_pre_generated_srs() { let tmp_dir = Builder::new().prefix("example").tempdir().unwrap(); // lets hope this link never rots - let target = "https://trusted-setup-halo2kzg.s3.eu-central-1.amazonaws.com/hermez-1"; + let target = "https://trusted-setup-halo2kzg.s3.eu-central-1.amazonaws.com/hermez-raw-1"; let response = reqwest::get(target).await.unwrap(); let fname = response @@ -547,7 +527,7 @@ mod tests { let mut dest = File::create(fname.clone()).unwrap(); let content = response.bytes().await.unwrap(); copy(&mut &content[..], &mut dest).unwrap(); - let res = load_params_kzg(fname); + let res = load_params::>(fname); assert!(res.is_ok()) } @@ -556,9 +536,9 @@ mod tests { let tmp_dir = Builder::new().prefix("example").tempdir().unwrap(); let fname = tmp_dir.path().join("kzg.params"); let srs = gen_srs::>(1); - let res = save_params_kzg(&fname, &srs); + let res = save_params::>(&fname, &srs); assert!(res.is_ok()); - let res = load_params_kzg(fname); + let res = load_params::>(fname); assert!(res.is_ok()) } }