fix: buffer data read and writes (#719)

src/circuit/modules/planner.rs

@@ -15,7 +15,7 @@ use halo2_proofs::{
         Instance, Selector, TableColumn,
     },
 };
-use log::{trace, warn};
+use log::{debug, trace};
 
 /// A simple [`FloorPlanner`] that performs minimal optimizations.
 #[derive(Debug)]
@@ -119,7 +119,7 @@ impl<'a, F: Field, CS: Assignment<F> + 'a + SyncDeps> Layouter<F> for ModuleLayo
                 Error::Synthesis
             })?;
             if !self.regions.contains_key(&index) {
-                warn!("spawning module {}", index)
+                debug!("spawning module {}", index)
             };
             self.current_module = index;
         }

src/commands.rs

@@ -203,7 +203,7 @@ impl<'source> FromPyObject<'source> for CalibrationTarget {
         }
     }
 }
-
+// not wasm
 use lazy_static::lazy_static;
 
 // if CARGO VERSION is 0.0.0 replace with "source - no compatibility guaranteed"

src/execute.rs

@@ -628,7 +628,7 @@ pub(crate) async fn gen_witness(
     );
 
     if let Some(output_path) = output {
-        serde_json::to_writer(&File::create(output_path)?, &witness)?;
+        witness.save(output_path)?;
     }
 
     // print the witness in debug

src/graph/input.rs

@@ -4,6 +4,7 @@ use crate::circuit::InputType;
 use crate::fieldutils::i128_to_felt;
 #[cfg(not(target_arch = "wasm32"))]
 use crate::tensor::Tensor;
+use crate::EZKL_BUF_CAPACITY;
 use halo2curves::bn256::Fr as Fp;
 #[cfg(not(target_arch = "wasm32"))]
 use postgres::{Client, NoTls};
@@ -15,6 +16,8 @@ use pyo3::types::PyDict;
 use pyo3::ToPyObject;
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::io::BufReader;
+use std::io::BufWriter;
 use std::io::Read;
 use std::panic::UnwindSafe;
 #[cfg(not(target_arch = "wasm32"))]
@@ -490,16 +493,20 @@ impl GraphData {
 
     /// Load the model input from a file
     pub fn from_path(path: std::path::PathBuf) -> Result<Self, Box<dyn std::error::Error>> {
-        let mut file = std::fs::File::open(path.clone())
-            .map_err(|_| format!("failed to open input at {}", path.display()))?;
-        let mut data = String::new();
-        file.read_to_string(&mut data)?;
-        serde_json::from_str(&data).map_err(|e| e.into())
+        let reader = std::fs::File::open(path)?;
+        let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, reader);
+        let mut buf = String::new();
+        reader.read_to_string(&mut buf)?;
+        let graph_input = serde_json::from_str(&buf)?;
+        Ok(graph_input)
     }
 
     /// Save the model input to a file
     pub fn save(&self, path: std::path::PathBuf) -> Result<(), Box<dyn std::error::Error>> {
-        serde_json::to_writer(std::fs::File::create(path)?, &self).map_err(|e| e.into())
+        // buf writer
+        let writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, std::fs::File::create(path)?);
+        serde_json::to_writer(writer, self)?;
+        Ok(())
     }
 
     ///

src/graph/mod.rs

@@ -28,7 +28,8 @@ use crate::circuit::{CheckMode, InputType};
 use crate::fieldutils::felt_to_f64;
 use crate::pfsys::PrettyElements;
 use crate::tensor::{Tensor, ValTensor};
-use crate::RunArgs;
+use crate::{RunArgs, EZKL_BUF_CAPACITY};
+
 use halo2_proofs::{
     circuit::Layouter,
     plonk::{Circuit, ConstraintSystem, Error as PlonkError},
@@ -37,7 +38,7 @@ use halo2curves::bn256::{self, Bn256, Fr as Fp, G1Affine};
 use halo2curves::ff::PrimeField;
 #[cfg(not(target_arch = "wasm32"))]
 use lazy_static::lazy_static;
-use log::{debug, error, info, trace, warn};
+use log::{debug, error, trace, warn};
 use maybe_rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
 pub use model::*;
 pub use node::*;
@@ -48,7 +49,6 @@ use pyo3::types::PyDict;
 #[cfg(feature = "python-bindings")]
 use pyo3::ToPyObject;
 use serde::{Deserialize, Serialize};
-use std::io::{Read, Write};
 use std::ops::Deref;
 use thiserror::Error;
 pub use utilities::*;
@@ -308,16 +308,20 @@ impl GraphWitness {
 
     /// Load the model input from a file
     pub fn from_path(path: std::path::PathBuf) -> Result<Self, Box<dyn std::error::Error>> {
-        let mut file = std::fs::File::open(path.clone())
+        let file = std::fs::File::open(path.clone())
             .map_err(|_| format!("failed to load model at {}", path.display()))?;
-        let mut data = String::new();
-        file.read_to_string(&mut data)?;
-        serde_json::from_str(&data).map_err(|e| e.into())
+
+        let reader = std::io::BufReader::with_capacity(*EZKL_BUF_CAPACITY, file);
+        serde_json::from_reader(reader).map_err(|e| e.into())
     }
 
     /// Save the model input to a file
     pub fn save(&self, path: std::path::PathBuf) -> Result<(), Box<dyn std::error::Error>> {
-        serde_json::to_writer(std::fs::File::create(path)?, &self).map_err(|e| e.into())
+        // use buf writer
+        let writer =
+            std::io::BufWriter::with_capacity(*EZKL_BUF_CAPACITY, std::fs::File::create(path)?);
+
+        serde_json::to_writer(writer, &self).map_err(|e| e.into())
     }
 
     ///
@@ -482,20 +486,23 @@ impl GraphSettings {
 
     /// save params to file
     pub fn save(&self, path: &std::path::PathBuf) -> Result<(), std::io::Error> {
-        let encoded = serde_json::to_string(&self)?;
-        let mut file = std::fs::File::create(path)?;
-        file.write_all(encoded.as_bytes())
+        // buf writer
+        let writer =
+            std::io::BufWriter::with_capacity(*EZKL_BUF_CAPACITY, std::fs::File::create(path)?);
+        serde_json::to_writer(writer, &self).map_err(|e| {
+            error!("failed to save settings file at {}", e);
+            std::io::Error::new(std::io::ErrorKind::Other, e)
+        })
     }
     /// load params from file
     pub fn load(path: &std::path::PathBuf) -> Result<Self, std::io::Error> {
-        let mut file = std::fs::File::open(path).map_err(|e| {
-            error!("failed to open settings file at {}", e);
-            e
-        })?;
-        let mut data = String::new();
-        file.read_to_string(&mut data)?;
-        let res = serde_json::from_str(&data)?;
-        Ok(res)
+        // buf reader
+        let reader =
+            std::io::BufReader::with_capacity(*EZKL_BUF_CAPACITY, std::fs::File::open(path)?);
+        serde_json::from_reader(reader).map_err(|e| {
+            error!("failed to load settings file at {}", e);
+            std::io::Error::new(std::io::ErrorKind::Other, e)
+        })
     }
 
     /// Export the ezkl configuration as json
@@ -591,7 +598,7 @@ impl GraphCircuit {
     ///
     pub fn save(&self, path: std::path::PathBuf) -> Result<(), Box<dyn std::error::Error>> {
         let f = std::fs::File::create(path)?;
-        let writer = std::io::BufWriter::new(f);
+        let writer = std::io::BufWriter::with_capacity(*EZKL_BUF_CAPACITY, f);
         bincode::serialize_into(writer, &self)?;
         Ok(())
     }
@@ -599,11 +606,10 @@ impl GraphCircuit {
     ///
     pub fn load(path: std::path::PathBuf) -> Result<Self, Box<dyn std::error::Error>> {
         // read bytes from file
-        let mut f = std::fs::File::open(&path)?;
-        let metadata = std::fs::metadata(&path)?;
-        let mut buffer = vec![0; metadata.len() as usize];
-        f.read_exact(&mut buffer)?;
-        let result = bincode::deserialize(&buffer)?;
+        let f = std::fs::File::open(&path)?;
+        let reader = std::io::BufReader::with_capacity(*EZKL_BUF_CAPACITY, f);
+        let result: GraphCircuit = bincode::deserialize_from(reader)?;
+
         Ok(result)
     }
 }
@@ -1520,7 +1526,7 @@ impl Circuit<Fp> for GraphCircuit {
         let circuit_size = CircuitSize::from_cs(cs, params.run_args.logrows);
 
         #[cfg(not(target_arch = "wasm32"))]
-        info!(
+        debug!(
             "circuit size: \n {}",
             circuit_size
                 .as_json()

src/graph/vars.rs

@@ -410,7 +410,7 @@ impl<F: PrimeField + TensorType + PartialOrd> ModelVars<F> {
         num_constants: usize,
         module_requires_fixed: bool,
     ) -> Self {
-        info!("number of blinding factors: {}", cs.blinding_factors());
+        debug!("number of blinding factors: {}", cs.blinding_factors());
 
         let advices = (0..3)
             .map(|_| VarTensor::new_advice(cs, logrows, num_inner_cols, var_len))

src/lib.rs

@@ -70,9 +70,32 @@ pub mod tensor;
 #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
 pub mod wasm;
 
+#[cfg(not(target_arch = "wasm32"))]
+use lazy_static::lazy_static;
+
 /// The denominator in the fixed point representation used when quantizing inputs
 pub type Scale = i32;
 
+#[cfg(not(target_arch = "wasm32"))]
+// Buf writer capacity
+lazy_static! {
+    /// The capacity of the buffer used for writing to disk
+    pub static ref EZKL_BUF_CAPACITY: usize = std::env::var("EZKL_BUF_CAPACITY")
+        .unwrap_or("8000".to_string())
+        .parse()
+        .unwrap();
+
+    /// The serialization format for the keys
+    pub static ref EZKL_KEY_FORMAT: String = std::env::var("EZKL_KEY_FORMAT")
+        .unwrap_or("raw-bytes".to_string());
+}
+
+#[cfg(target_arch = "wasm32")]
+const EZKL_KEY_FORMAT: &str = "raw-bytes";
+
+#[cfg(target_arch = "wasm32")]
+const EZKL_BUF_CAPACITY: &usize = &8000;
+
 /// Parameters specific to a proving run
 #[derive(Debug, Args, Deserialize, Serialize, Clone, PartialEq, PartialOrd)]
 pub struct RunArgs {

src/pfsys/mod.rs

@@ -8,6 +8,7 @@ use crate::circuit::CheckMode;
 use crate::graph::GraphWitness;
 use crate::pfsys::evm::aggregation::PoseidonTranscript;
 use crate::tensor::TensorType;
+use crate::{EZKL_BUF_CAPACITY, EZKL_KEY_FORMAT};
 use clap::ValueEnum;
 use halo2_proofs::circuit::Value;
 use halo2_proofs::plonk::{
@@ -40,24 +41,17 @@ use std::ops::Deref;
 use std::path::PathBuf;
 use thiserror::Error as thisError;
 
-// not wasm
-#[cfg(not(target_arch = "wasm32"))]
-use lazy_static::lazy_static;
-
 use halo2curves::bn256::{Bn256, Fr, G1Affine};
 
-#[cfg(not(target_arch = "wasm32"))]
-// Buf writer capacity
-lazy_static! {
-    static ref EZKL_BUF_CAPACITY: usize = std::env::var("EZKL_BUF_CAPACITY")
-        .unwrap_or("8000".to_string())
-        .parse()
-        .unwrap();
+fn serde_format_from_str(s: &str) -> halo2_proofs::SerdeFormat {
+    match s {
+        "processed" => halo2_proofs::SerdeFormat::Processed,
+        "raw-bytes-unchecked" => halo2_proofs::SerdeFormat::RawBytesUnchecked,
+        "raw-bytes" => halo2_proofs::SerdeFormat::RawBytes,
+        _ => panic!("invalid serde format"),
+    }
 }
 
-#[cfg(target_arch = "wasm32")]
-const EZKL_BUF_CAPACITY: &usize = &8000;
-
 #[allow(missing_docs)]
 #[derive(
     ValueEnum, Copy, Clone, Default, Debug, PartialEq, Eq, Deserialize, Serialize, PartialOrd,
@@ -344,8 +338,10 @@ where
         <C as CurveAffine>::ScalarExt: FromUniformBytes<64>,
     {
         trace!("reading proof");
-        let data = std::fs::read_to_string(proof_path)?;
-        serde_json::from_str(&data).map_err(|e| e.into())
+        let file = std::fs::File::open(proof_path)?;
+        let reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, file);
+        let proof: Self = serde_json::from_reader(reader)?;
+        Ok(proof)
     }
 }
 
@@ -699,7 +695,7 @@ where
     let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, f);
     VerifyingKey::<Scheme::Curve>::read::<_, C>(
         &mut reader,
-        halo2_proofs::SerdeFormat::RawBytes,
+        serde_format_from_str(&EZKL_KEY_FORMAT),
         params,
     )
     .map_err(Box::<dyn Error>::from)
@@ -721,7 +717,7 @@ where
     let mut reader = BufReader::with_capacity(*EZKL_BUF_CAPACITY, f);
     ProvingKey::<Scheme::Curve>::read::<_, C>(
         &mut reader,
-        halo2_proofs::SerdeFormat::RawBytes,
+        serde_format_from_str(&EZKL_KEY_FORMAT),
         params,
     )
     .map_err(Box::<dyn Error>::from)
@@ -730,7 +726,7 @@ where
 /// Saves a [ProvingKey] to `path`.
 pub fn save_pk<Scheme: CommitmentScheme>(
     path: &PathBuf,
-    vk: &ProvingKey<Scheme::Curve>,
+    pk: &ProvingKey<Scheme::Curve>,
 ) -> Result<(), io::Error>
 where
     Scheme::Curve: SerdeObject + CurveAffine,
@@ -739,7 +735,7 @@ where
     info!("saving proving key 💾");
     let f = File::create(path)?;
     let mut writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, f);
-    vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?;
+    pk.write(&mut writer, serde_format_from_str(&EZKL_KEY_FORMAT))?;
     writer.flush()?;
     Ok(())
 }
@@ -756,7 +752,7 @@ where
     info!("saving verification key 💾");
     let f = File::create(path)?;
     let mut writer = BufWriter::with_capacity(*EZKL_BUF_CAPACITY, f);
-    vk.write(&mut writer, halo2_proofs::SerdeFormat::RawBytes)?;
+    vk.write(&mut writer, serde_format_from_str(&EZKL_KEY_FORMAT))?;
     writer.flush()?;
     Ok(())
 }