fix: buffer data read and writes (#719)

src/circuit/modules/planner.rs

@@ -15,7 +15,7 @@ use halo2_proofs::{
         Instance, Selector, TableColumn,
     },
 };
-use log::{trace, warn};
+use log::{debug, trace};
 
 /// A simple [`FloorPlanner`] that performs minimal optimizations.
 #[derive(Debug)]
@@ -119,7 +119,7 @@ impl<'a, F: Field, CS: Assignment<F> + 'a + SyncDeps> Layouter<F> for ModuleLayo
                 Error::Synthesis
             })?;
             if !self.regions.contains_key(&index) {
-                warn!("spawning module {}", index)
+                debug!("spawning module {}", index)
             };
             self.current_module = index;
         }

src/graph/input.rs

@@ -4,6 +4,7 @@ use crate::circuit::InputType;
 use crate::fieldutils::i128_to_felt;
 #[cfg(not(target_arch = "wasm32"))]
 use crate::tensor::Tensor;
+use crate::EZKL_BUF_CAPACITY;
 use halo2curves::bn256::Fr as Fp;
 #[cfg(not(target_arch = "wasm32"))]
 use postgres::{Client, NoTls};
@@ -15,6 +16,8 @@ use pyo3::types::PyDict;
 use pyo3::ToPyObject;
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::io::BufReader;
+use std::io::BufWriter;
 use std::io::Read;
 use std::panic::UnwindSafe;
 #[cfg(not(target_arch = "wasm32"))]
@@ -490,16 +493,20 @@ impl GraphData {
 
     /// Load the model input from a file
     pub fn from_path(path: std::path::PathBuf) -> Result<Self, Box<dyn std::error::Error>> {
-        let mut file = std::fs::File::open(path.clone())
-            .map_err(|_| format!("failed to open input at {}", path.display()))?;
-        let mut data = String::new();
-        file.read_to_string(&mut data)?;
-        serde_json::from_str(&data).map_err(|e| e.into())
+        let reader = std::fs::File::open(path)?;
+        let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, reader);
+        let mut buf = String::new();
+        reader.read_to_string(&mut buf)?;
+        let graph_input = serde_json::from_str(&buf)?;
+        Ok(graph_input)
     }
 
     /// Save the model input to a file
     pub fn save(&self, path: std::path::PathBuf) -> Result<(), Box<dyn std::error::Error>> {
-        serde_json::to_writer(std::fs::File::create(path)?, &self).map_err(|e| e.into())
+        // buf writer
+        let writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, std::fs::File::create(path)?);
+        serde_json::to_writer(writer, self)?;
+        Ok(())
     }
 
     ///

src/graph/mod.rs

@@ -38,7 +38,7 @@ use halo2curves::bn256::{self, Bn256, Fr as Fp, G1Affine};
 use halo2curves::ff::PrimeField;
 #[cfg(not(target_arch = "wasm32"))]
 use lazy_static::lazy_static;
-use log::{debug, error, info, trace, warn};
+use log::{debug, error, trace, warn};
 use maybe_rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
 pub use model::*;
 pub use node::*;
@@ -1526,7 +1526,7 @@ impl Circuit<Fp> for GraphCircuit {
         let circuit_size = CircuitSize::from_cs(cs, params.run_args.logrows);
 
         #[cfg(not(target_arch = "wasm32"))]
-        info!(
+        debug!(
             "circuit size: \n {}",
             circuit_size
                 .as_json()

src/graph/vars.rs

@@ -410,7 +410,7 @@ impl<F: PrimeField + TensorType + PartialOrd> ModelVars<F> {
         num_constants: usize,
         module_requires_fixed: bool,
     ) -> Self {
-        info!("number of blinding factors: {}", cs.blinding_factors());
+        debug!("number of blinding factors: {}", cs.blinding_factors());
 
         let advices = (0..3)
             .map(|_| VarTensor::new_advice(cs, logrows, num_inner_cols, var_len))

src/lib.rs

@@ -31,16 +31,12 @@
 use circuit::{table::Range, CheckMode, Tolerance};
 use clap::Args;
 use graph::Visibility;
-
 use serde::{Deserialize, Serialize};
 
-#[cfg(not(target_arch = "wasm32"))]
-use lazy_static::lazy_static;
-
 /// Methods for configuring tensor operations and assigning values to them in a Halo2 circuit.
 pub mod circuit;
-#[cfg(not(target_arch = "wasm32"))]
 /// CLI commands.
+#[cfg(not(target_arch = "wasm32"))]
 pub mod commands;
 #[cfg(not(target_arch = "wasm32"))]
 // abigen doesn't generate docs for this module
@@ -74,6 +70,9 @@ pub mod tensor;
 #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
 pub mod wasm;
 
+#[cfg(not(target_arch = "wasm32"))]
+use lazy_static::lazy_static;
+
 /// The denominator in the fixed point representation used when quantizing inputs
 pub type Scale = i32;
 
@@ -85,8 +84,15 @@ lazy_static! {
         .unwrap_or("8000".to_string())
         .parse()
         .unwrap();
+
+    /// The serialization format for the keys
+    pub static ref EZKL_KEY_FORMAT: String = std::env::var("EZKL_KEY_FORMAT")
+        .unwrap_or("raw-bytes".to_string());
 }
 
+#[cfg(target_arch = "wasm32")]
+const EZKL_KEY_FORMAT: &str = "raw-bytes";
+
 #[cfg(target_arch = "wasm32")]
 const EZKL_BUF_CAPACITY: &usize = &8000;
 

src/pfsys/mod.rs

@@ -8,7 +8,7 @@ use crate::circuit::CheckMode;
 use crate::graph::GraphWitness;
 use crate::pfsys::evm::aggregation::PoseidonTranscript;
 use crate::tensor::TensorType;
-use crate::EZKL_BUF_CAPACITY;
+use crate::{EZKL_BUF_CAPACITY, EZKL_KEY_FORMAT};
 use clap::ValueEnum;
 use halo2_proofs::circuit::Value;
 use halo2_proofs::plonk::{
@@ -43,6 +43,15 @@ use thiserror::Error as thisError;
 
 use halo2curves::bn256::{Bn256, Fr, G1Affine};
 
+fn serde_format_from_str(s: &str) -> halo2_proofs::SerdeFormat {
+    match s {
+        "processed" => halo2_proofs::SerdeFormat::Processed,
+        "raw-bytes-unchecked" => halo2_proofs::SerdeFormat::RawBytesUnchecked,
+        "raw-bytes" => halo2_proofs::SerdeFormat::RawBytes,
+        _ => panic!("invalid serde format"),
+    }
+}
+
 #[allow(missing_docs)]
 #[derive(
     ValueEnum, Copy, Clone, Default, Debug, PartialEq, Eq, Deserialize, Serialize, PartialOrd,
@@ -686,7 +695,7 @@ where
     let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, f);
     VerifyingKey::<Scheme::Curve>::read::<_, C>(
         &mut reader,
-        halo2_proofs::SerdeFormat::RawBytes,
+        serde_format_from_str(&EZKL_KEY_FORMAT),
         params,
     )
     .map_err(Box::<dyn Error>::from)
@@ -708,7 +717,7 @@ where
     let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, f);
     ProvingKey::<Scheme::Curve>::read::<_, C>(
         &mut reader,
-        halo2_proofs::SerdeFormat::RawBytes,
+        serde_format_from_str(&EZKL_KEY_FORMAT),
         params,
     )
     .map_err(Box::<dyn Error>::from)
@@ -717,7 +726,7 @@ where
 /// Saves a [ProvingKey] to `path`.
 pub fn save_pk<Scheme: CommitmentScheme>(
     path: &PathBuf,
-    vk: &ProvingKey<Scheme::Curve>,
+    pk: &ProvingKey<Scheme::Curve>,
 ) -> Result<(), io::Error>
 where
     Scheme::Curve: SerdeObject + CurveAffine,
@@ -726,7 +735,7 @@ where
     info!("saving proving key 💾");
     let f = File::create(path)?;
     let mut writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, f);
-    vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?;
+    pk.write(&mut writer, serde_format_from_str(&EZKL_KEY_FORMAT))?;
     writer.flush()?;
     Ok(())
 }
@@ -743,7 +752,7 @@ where
     info!("saving verification key 💾");
     let f = File::create(path)?;
     let mut writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, f);
-    vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?;
+    vk.write(&mut writer, serde_format_from_str(&EZKL_KEY_FORMAT))?;
     writer.flush()?;
     Ok(())
 }