feat: wasm felt to little endian string (#729)

---------

Co-authored-by: Alexander Camuto <45801863+alexander-camuto@users.noreply.github.com>

.github/workflows/rust.yml

@@ -575,7 +575,7 @@ jobs:
       - name: Install Anvil
         run: cargo install --git https://github.com/foundry-rs/foundry --rev b320f350156a0fb15c2eb13dc380deb2367c4474 --profile local --locked anvil --force
       - name: Build python ezkl
-        run: source .env/bin/activate; maturin develop --features python-bindings --release
+        run: source .env/bin/activate; unset CONDA_PREFIX; maturin develop --features python-bindings --release
       - name: Run pytest
         run: source .env/bin/activate; pytest -vv
 
@@ -599,7 +599,7 @@ jobs:
       - name: Setup Virtual Env and Install python dependencies
         run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
       - name: Build python ezkl
-        run: source .env/bin/activate; maturin develop --features python-bindings --release
+        run: source .env/bin/activate; unset CONDA_PREFIX; maturin develop --features python-bindings --release
       - name: Div rebase
         run: source .env/bin/activate; cargo nextest run --release --verbose tests::accuracy_measurement_div_rebase_
       - name: Public inputs
@@ -634,7 +634,7 @@ jobs:
       - name: Setup Virtual Env and Install python dependencies
         run: python -m venv .env; source .env/bin/activate; pip install -r requirements.txt;
       - name: Build python ezkl
-        run: source .env/bin/activate; maturin develop --features python-bindings --release
+        run: source .env/bin/activate; unset CONDA_PREFIX; maturin develop --features python-bindings --release
       # - name: authenticate-kaggle-cli
       #   shell: bash
       #   env:

src/circuit/ops/chip.rs

@@ -198,6 +198,9 @@ pub struct BaseConfig<F: PrimeField + TensorType + PartialOrd> {
     pub lookup_input: VarTensor,
     /// the (currently singular) output of the accumulated operations.
     pub output: VarTensor,
+    /// The VarTensor reserved for dynamic lookup operations (could be an element of inputs or the same as output)
+    /// Note that you should be careful to ensure that the lookup_output is not simultaneously assigned to by other non-lookup operations eg. in the case of composite ops.
+    pub dynamic_lookup_tables: Vec<VarTensor>,
     /// the VarTensor reserved for lookup operations (could be an element of inputs or the same as output)
     /// Note that you should be careful to ensure that the lookup_output is not simultaneously assigned to by other non-lookup operations eg. in the case of composite ops.
     pub lookup_output: VarTensor,
@@ -207,6 +210,10 @@ pub struct BaseConfig<F: PrimeField + TensorType + PartialOrd> {
     pub selectors: BTreeMap<(BaseOp, usize, usize), Selector>,
     /// [Selector]s generated when configuring the layer. We use a [BTreeMap] as we expect to configure many lookup ops.
     pub lookup_selectors: BTreeMap<(LookupOp, usize, usize), Selector>,
+    /// [Selector]s generated when configuring the layer. We use a [BTreeMap] as we expect to configure many dynamic lookup ops.
+    pub dynamic_lookup_selectors: BTreeMap<(usize, usize), Vec<Selector>>,
+    ///
+    pub dynamic_table_selectors: Vec<Selector>,
     ///
     pub tables: BTreeMap<LookupOp, Table<F>>,
     ///
@@ -228,9 +235,12 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
             lookup_input: dummy_var.clone(),
             output: dummy_var.clone(),
             lookup_output: dummy_var.clone(),
+            dynamic_lookup_tables: vec![VarTensor::dummy(col_size, 2), dummy_var.clone()],
             lookup_index: dummy_var,
             selectors: BTreeMap::new(),
             lookup_selectors: BTreeMap::new(),
+            dynamic_lookup_selectors: BTreeMap::new(),
+            dynamic_table_selectors: vec![],
             range_check_selectors: BTreeMap::new(),
             tables: BTreeMap::new(),
             range_checks: BTreeMap::new(),
@@ -376,10 +386,13 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
             selectors,
             lookup_selectors: BTreeMap::new(),
             range_check_selectors: BTreeMap::new(),
+            dynamic_lookup_selectors: BTreeMap::new(),
             inputs: inputs.to_vec(),
             lookup_input: VarTensor::Empty,
             lookup_output: VarTensor::Empty,
             lookup_index: VarTensor::Empty,
+            dynamic_table_selectors: vec![],
+            dynamic_lookup_tables: vec![],
             tables: BTreeMap::new(),
             range_checks: BTreeMap::new(),
             output: output.clone(),
@@ -403,8 +416,6 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
     where
         F: Field,
     {
-        let mut selectors = BTreeMap::new();
-
         if !index.is_advice() {
             return Err("wrong input type for lookup index".into());
         }
@@ -514,10 +525,10 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
                         res
                     });
                 }
-                selectors.insert((nl.clone(), x, y), multi_col_selector);
+                self.lookup_selectors
+                    .insert((nl.clone(), x, y), multi_col_selector);
             }
         }
-        self.lookup_selectors.extend(selectors);
         // if we haven't previously initialized the input/output, do so now
         if let VarTensor::Empty = self.lookup_input {
             debug!("assigning lookup input");
@@ -534,6 +545,85 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
         Ok(())
     }
 
+    /// Configures and creates lookup selectors
+    #[allow(clippy::too_many_arguments)]
+    pub fn configure_dynamic_lookup(
+        &mut self,
+        cs: &mut ConstraintSystem<F>,
+        lookups: &[VarTensor; 2],
+        tables: &[VarTensor; 2],
+    ) -> Result<(), Box<dyn Error>>
+    where
+        F: Field,
+    {
+        for l in lookups.iter() {
+            if !l.is_advice() {
+                return Err("wrong input type for dynamic lookup".into());
+            }
+        }
+
+        for t in tables.iter() {
+            if !t.is_advice() || t.num_blocks() > 1 || t.num_inner_cols() > 1 {
+                return Err("wrong table type for dynamic lookup".into());
+            }
+        }
+
+        let one = Expression::Constant(F::ONE);
+
+        let s_ltable = cs.complex_selector();
+
+        for x in 0..lookups[0].num_blocks() {
+            for y in 0..lookups[0].num_inner_cols() {
+                let s_lookup = cs.complex_selector();
+
+                cs.lookup_any("lookup", |cs| {
+                    let s_lookupq = cs.query_selector(s_lookup);
+                    let mut expression = vec![];
+                    let s_ltableq = cs.query_selector(s_ltable);
+                    let mut lookup_queries = vec![one.clone()];
+
+                    for lookup in lookups {
+                        lookup_queries.push(match lookup {
+                            VarTensor::Advice { inner: advices, .. } => {
+                                cs.query_advice(advices[x][y], Rotation(0))
+                            }
+                            _ => unreachable!(),
+                        });
+                    }
+
+                    let mut table_queries = vec![one.clone()];
+                    for table in tables {
+                        table_queries.push(match table {
+                            VarTensor::Advice { inner: advices, .. } => {
+                                cs.query_advice(advices[0][0], Rotation(0))
+                            }
+                            _ => unreachable!(),
+                        });
+                    }
+
+                    let lhs = lookup_queries.into_iter().map(|c| c * s_lookupq.clone());
+                    let rhs = table_queries.into_iter().map(|c| c * s_ltableq.clone());
+                    expression.extend(lhs.zip(rhs));
+
+                    expression
+                });
+                self.dynamic_lookup_selectors
+                    .entry((x, y))
+                    .or_default()
+                    .push(s_lookup);
+            }
+        }
+        self.dynamic_table_selectors.push(s_ltable);
+
+        // if we haven't previously initialized the input/output, do so now
+        if self.dynamic_lookup_tables.is_empty() {
+            debug!("assigning dynamic lookup table");
+            self.dynamic_lookup_tables = tables.to_vec();
+        }
+
+        Ok(())
+    }
+
     /// Configures and creates lookup selectors
     #[allow(clippy::too_many_arguments)]
     pub fn configure_range_check(
@@ -547,8 +637,6 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
     where
         F: Field,
     {
-        let mut selectors = BTreeMap::new();
-
         if !input.is_advice() {
             return Err("wrong input type for lookup input".into());
         }
@@ -620,10 +708,10 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
                         res
                     });
                 }
-                selectors.insert((range, x, y), multi_col_selector);
+                self.range_check_selectors
+                    .insert((range, x, y), multi_col_selector);
             }
         }
-        self.range_check_selectors.extend(selectors);
         // if we haven't previously initialized the input/output, do so now
         if let VarTensor::Empty = self.lookup_input {
             debug!("assigning lookup input");

src/circuit/ops/layouts.rs

@@ -915,6 +915,78 @@ fn one_hot<F: PrimeField + TensorType + PartialOrd>(
     Ok(assigned_output)
 }
 
+/// Dynamic lookup
+pub fn dynamic_lookup<F: PrimeField + TensorType + PartialOrd>(
+    config: &BaseConfig<F>,
+    region: &mut RegionCtx<F>,
+    lookups: &[ValTensor<F>; 2],
+    tables: &[ValTensor<F>; 2],
+) -> Result<(ValTensor<F>, ValTensor<F>), Box<dyn Error>> {
+    // if not all lookups same length err
+    if lookups[0].len() != lookups[1].len() {
+        return Err("lookups must be same length".into());
+    }
+
+    // if not all inputs same length err
+    if tables[0].len() != tables[1].len() {
+        return Err("inputs must be same length".into());
+    }
+
+    // now assert the inputs of a smaller length than the lookups
+    if tables[0].len() > tables[0].len() {
+        return Err("inputs must be smaller length than dynamic lookups".into());
+    }
+
+    let (lookup_0, lookup_1) = (lookups[0].clone(), lookups[1].clone());
+    let (table_0, table_1) = (tables[0].clone(), tables[1].clone());
+
+    let table_0 = region.assign_dynamic_lookup(&config.dynamic_lookup_tables[0], &table_0)?;
+    let _table_1 = region.assign_dynamic_lookup(&config.dynamic_lookup_tables[1], &table_1)?;
+    let table_len = table_0.len();
+
+    let lookup_0 = region.assign(&config.inputs[0], &lookup_0)?;
+    let lookup_1 = region.assign(&config.inputs[1], &lookup_1)?;
+
+    let lookup_len = lookup_0.len();
+
+    if !region.is_dummy() {
+        (0..table_len)
+            .map(|i| {
+                let dynamic_lookup_index = region.dynamic_lookup_index();
+                let table_selector = config.dynamic_table_selectors[dynamic_lookup_index];
+                let (_, _, z) = config.dynamic_lookup_tables[0]
+                    .cartesian_coord(region.dynamic_lookup_col_coord() + i);
+                region.enable(Some(&table_selector), z)?;
+                Ok(())
+            })
+            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
+    }
+
+    if !region.is_dummy() {
+        // Enable the selectors
+        (0..lookup_len)
+            .map(|i| {
+                let (x, y, z) = config.inputs[0].cartesian_coord(region.linear_coord() + i);
+                let dynamic_lookup_index = region.dynamic_lookup_index();
+                let lookup_selector = config
+                    .dynamic_lookup_selectors
+                    .get(&(x, y))
+                    .ok_or("missing selectors")?[dynamic_lookup_index];
+
+                region.enable(Some(&lookup_selector), z)?;
+
+                Ok(())
+            })
+            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
+    }
+
+    region.increment_dynamic_lookup_col_coord(table_len);
+    region.increment_dynamic_lookup_index(1);
+    region.increment(lookup_len);
+
+    Ok((lookup_0, lookup_1))
+}
+
 /// One hot accumulated layout
 pub fn one_hot_axis<F: PrimeField + TensorType + PartialOrd>(
     config: &BaseConfig<F>,

src/circuit/ops/region.rs

@@ -20,6 +20,33 @@ use portable_atomic::AtomicI128 as AtomicInt;
 
 use super::lookup::LookupOp;
 
+/// Dynamic lookup index
+#[derive(Clone, Debug, Default)]
+pub struct DynamicLookupIndex {
+    lookup_index: usize,
+    col_coord: usize,
+}
+
+impl DynamicLookupIndex {
+    /// Create a new dynamic lookup index
+    pub fn new(lookup_index: usize, col_coord: usize) -> DynamicLookupIndex {
+        DynamicLookupIndex {
+            lookup_index,
+            col_coord,
+        }
+    }
+
+    /// Get the lookup index
+    pub fn lookup_index(&self) -> usize {
+        self.lookup_index
+    }
+
+    /// Get the column coord
+    pub fn col_coord(&self) -> usize {
+        self.col_coord
+    }
+}
+
 /// Region error
 #[derive(Debug, thiserror::Error)]
 pub enum RegionError {
@@ -66,6 +93,7 @@ pub struct RegionCtx<'a, F: PrimeField + TensorType + PartialOrd> {
     linear_coord: usize,
     num_inner_cols: usize,
     total_constants: usize,
+    dynamic_lookup_index: DynamicLookupIndex,
     used_lookups: HashSet<LookupOp>,
     used_range_checks: HashSet<Range>,
     max_lookup_inputs: i128,
@@ -80,6 +108,16 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         self.total_constants += n;
     }
 
+    ///
+    pub fn increment_dynamic_lookup_index(&mut self, n: usize) {
+        self.dynamic_lookup_index.lookup_index += n;
+    }
+
+    ///
+    pub fn increment_dynamic_lookup_col_coord(&mut self, n: usize) {
+        self.dynamic_lookup_index.col_coord += n;
+    }
+
     ///
     pub fn throw_range_check_error(&self) -> bool {
         self.throw_range_check_error
@@ -96,6 +134,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
             row,
             linear_coord,
             total_constants: 0,
+            dynamic_lookup_index: DynamicLookupIndex::default(),
             used_lookups: HashSet::new(),
             used_range_checks: HashSet::new(),
             max_lookup_inputs: 0,
@@ -109,6 +148,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         region: Option<RefCell<Region<'a, F>>>,
         row: usize,
         num_inner_cols: usize,
+        dynamic_lookup_index: DynamicLookupIndex,
     ) -> RegionCtx<'a, F> {
         let linear_coord = row * num_inner_cols;
         RegionCtx {
@@ -117,6 +157,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
             linear_coord,
             row,
             total_constants: 0,
+            dynamic_lookup_index,
             used_lookups: HashSet::new(),
             used_range_checks: HashSet::new(),
             max_lookup_inputs: 0,
@@ -141,6 +182,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
             linear_coord,
             row,
             total_constants: 0,
+            dynamic_lookup_index: DynamicLookupIndex::default(),
             used_lookups: HashSet::new(),
             used_range_checks: HashSet::new(),
             max_lookup_inputs: 0,
@@ -156,6 +198,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         linear_coord: usize,
         total_constants: usize,
         num_inner_cols: usize,
+        dynamic_lookup_index: DynamicLookupIndex,
         used_lookups: HashSet<LookupOp>,
         used_range_checks: HashSet<Range>,
         throw_range_check_error: bool,
@@ -167,6 +210,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
             linear_coord,
             row,
             total_constants,
+            dynamic_lookup_index,
             used_lookups,
             used_range_checks,
             max_lookup_inputs: 0,
@@ -227,6 +271,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         let min_lookup_inputs = AtomicInt::new(self.min_lookup_inputs());
         let lookups = Arc::new(Mutex::new(self.used_lookups.clone()));
         let range_checks = Arc::new(Mutex::new(self.used_range_checks.clone()));
+        let dynamic_lookup_index = Arc::new(Mutex::new(self.dynamic_lookup_index.clone()));
 
         *output = output
             .par_enum_map(|idx, _| {
@@ -242,6 +287,7 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
                     starting_linear_coord,
                     starting_constants,
                     self.num_inner_cols,
+                    DynamicLookupIndex::default(),
                     HashSet::new(),
                     HashSet::new(),
                     self.throw_range_check_error,
@@ -265,6 +311,10 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
                 lookups.extend(local_reg.used_lookups());
                 let mut range_checks = range_checks.lock().unwrap();
                 range_checks.extend(local_reg.used_range_checks());
+                let mut dynamic_lookup_index = dynamic_lookup_index.lock().unwrap();
+                dynamic_lookup_index.lookup_index += local_reg.dynamic_lookup_index.lookup_index;
+                dynamic_lookup_index.col_coord += local_reg.dynamic_lookup_index.col_coord;
+
                 res
             })
             .map_err(|e| {
@@ -293,6 +343,20 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
             .map_err(|e| {
                 RegionError::from(format!("dummy_loop: failed to get range checks: {:?}", e))
             })?;
+        self.dynamic_lookup_index = Arc::try_unwrap(dynamic_lookup_index)
+            .map_err(|e| {
+                RegionError::from(format!(
+                    "dummy_loop: failed to get dynamic lookup index: {:?}",
+                    e
+                ))
+            })?
+            .into_inner()
+            .map_err(|e| {
+                RegionError::from(format!(
+                    "dummy_loop: failed to get dynamic lookup index: {:?}",
+                    e
+                ))
+            })?;
 
         Ok(())
     }
@@ -363,6 +427,16 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         self.total_constants
     }
 
+    /// Get the dynamic lookup index
+    pub fn dynamic_lookup_index(&self) -> usize {
+        self.dynamic_lookup_index.lookup_index
+    }
+
+    /// Get the dynamic lookup column coordinate
+    pub fn dynamic_lookup_col_coord(&self) -> usize {
+        self.dynamic_lookup_index.col_coord
+    }
+
     /// get used lookups
     pub fn used_lookups(&self) -> HashSet<LookupOp> {
         self.used_lookups.clone()
@@ -412,6 +486,24 @@ impl<'a, F: PrimeField + TensorType + PartialOrd> RegionCtx<'a, F> {
         }
     }
 
+    /// Assign a valtensor to a vartensor
+    pub fn assign_dynamic_lookup(
+        &mut self,
+        var: &VarTensor,
+        values: &ValTensor<F>,
+    ) -> Result<ValTensor<F>, Error> {
+        self.total_constants += values.num_constants();
+        if let Some(region) = &self.region {
+            var.assign(
+                &mut region.borrow_mut(),
+                self.dynamic_lookup_col_coord(),
+                values,
+            )
+        } else {
+            Ok(values.clone())
+        }
+    }
+
     /// Assign a valtensor to a vartensor
     pub fn assign_with_omissions(
         &mut self,

src/circuit/tests.rs

@@ -1574,6 +1574,142 @@ mod add {
     }
 }
 
+#[cfg(test)]
+mod dynamic_lookup {
+    use super::*;
+
+    const K: usize = 6;
+    const LEN: usize = 4;
+    const NUM_LOOP: usize = 5;
+
+    #[derive(Clone)]
+    struct MyCircuit<F: PrimeField + TensorType + PartialOrd> {
+        tables: [[ValTensor<F>; 2]; NUM_LOOP],
+        lookups: [[ValTensor<F>; 2]; NUM_LOOP],
+        _marker: PhantomData<F>,
+    }
+
+    impl Circuit<F> for MyCircuit<F> {
+        type Config = BaseConfig<F>;
+        type FloorPlanner = SimpleFloorPlanner;
+        type Params = TestParams;
+
+        fn without_witnesses(&self) -> Self {
+            self.clone()
+        }
+
+        fn configure(cs: &mut ConstraintSystem<F>) -> Self::Config {
+            let a = VarTensor::new_advice(cs, K, 2, LEN);
+            let b = VarTensor::new_advice(cs, K, 2, LEN);
+            let c: VarTensor = VarTensor::new_advice(cs, K, 2, LEN);
+
+            let d = VarTensor::new_advice(cs, K, 1, LEN);
+            let e = VarTensor::new_advice(cs, K, 1, LEN);
+
+            let mut config =
+                Self::Config::configure(cs, &[a.clone(), b.clone()], &c, CheckMode::SAFE);
+            for _ in 0..NUM_LOOP {
+                config
+                    .configure_dynamic_lookup(cs, &[a.clone(), b.clone()], &[d.clone(), e.clone()])
+                    .unwrap();
+            }
+
+            config
+        }
+
+        fn synthesize(
+            &self,
+            config: Self::Config,
+            mut layouter: impl Layouter<F>,
+        ) -> Result<(), Error> {
+            layouter
+                .assign_region(
+                    || "",
+                    |region| {
+                        let mut region = RegionCtx::new(region, 0, 1);
+                        for i in 0..NUM_LOOP {
+                            layouts::dynamic_lookup(
+                                &config,
+                                &mut region,
+                                &self.lookups[i],
+                                &self.tables[i],
+                            )
+                            .map_err(|_| Error::Synthesis)?;
+                        }
+                        assert_eq!(
+                            region.dynamic_lookup_col_coord(),
+                            NUM_LOOP * self.tables[0][0].len()
+                        );
+                        assert_eq!(region.dynamic_lookup_index(), NUM_LOOP);
+
+                        Ok(())
+                    },
+                )
+                .unwrap();
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn dynamiclookupcircuit() {
+        // parameters
+        let tables = (0..NUM_LOOP)
+            .map(|loop_idx| {
+                [
+                    ValTensor::from(Tensor::from(
+                        (0..LEN).map(|i| Value::known(F::from((i * loop_idx) as u64 + 1))),
+                    )),
+                    ValTensor::from(Tensor::from(
+                        (0..LEN).map(|i| Value::known(F::from((loop_idx * i * i) as u64 + 1))),
+                    )),
+                ]
+            })
+            .collect::<Vec<_>>();
+
+        let lookups = (0..NUM_LOOP)
+            .map(|loop_idx| {
+                [
+                    ValTensor::from(Tensor::from(
+                        (0..3).map(|i| Value::known(F::from((i * loop_idx) as u64 + 1))),
+                    )),
+                    ValTensor::from(Tensor::from(
+                        (0..3).map(|i| Value::known(F::from((loop_idx * i * i) as u64 + 1))),
+                    )),
+                ]
+            })
+            .collect::<Vec<_>>();
+
+        let circuit = MyCircuit::<F> {
+            tables: tables.clone().try_into().unwrap(),
+            lookups: lookups.try_into().unwrap(),
+            _marker: PhantomData,
+        };
+
+        let prover = MockProver::run(K as u32, &circuit, vec![]).unwrap();
+        prover.assert_satisfied();
+
+        let lookups = (0..NUM_LOOP)
+            .map(|loop_idx| {
+                [
+                    ValTensor::from(Tensor::from(
+                        (0..2).map(|i| Value::known(F::from((i * loop_idx) as u64 + 1))),
+                    )),
+                    ValTensor::from(Tensor::from((0..2).map(|_| Value::known(F::from(10000))))),
+                ]
+            })
+            .collect::<Vec<_>>();
+
+        let circuit = MyCircuit::<F> {
+            tables: tables.try_into().unwrap(),
+            lookups: lookups.try_into().unwrap(),
+            _marker: PhantomData,
+        };
+
+        let prover = MockProver::run(K as u32, &circuit, vec![]).unwrap();
+        assert!(prover.verify().is_err());
+    }
+}
+
 #[cfg(test)]
 mod add_with_overflow {
     use super::*;

src/execute.rs

@@ -962,7 +962,7 @@ pub(crate) fn calibrate(
             .max()
             .unwrap_or(0);
 
-        let res = circuit.calibrate_from_min_max(
+        let res = circuit.calc_min_logrows(
             (min_lookup_range, max_lookup_range),
             max_range_size,
             max_logrows,

src/graph/mod.rs

@@ -449,6 +449,10 @@ pub struct GraphSettings {
     pub total_assignments: usize,
     /// total const size
     pub total_const_size: usize,
+    /// total dynamic column size
+    pub total_dynamic_col_size: usize,
+    /// number of dynamic lookups
+    pub num_dynamic_lookups: usize,
     /// the shape of public inputs to the model (in order of appearance)
     pub model_instance_shapes: Vec<Vec<usize>>,
     /// model output scales
@@ -478,6 +482,10 @@ impl GraphSettings {
             .ceil() as u32
     }
 
+    fn dynamic_lookup_logrows(&self) -> u32 {
+        (self.total_dynamic_col_size as f64).log2().ceil() as u32
+    }
+
     fn module_constraint_logrows(&self) -> u32 {
         (self.module_sizes.max_constraints() as f64).log2().ceil() as u32
     }
@@ -570,6 +578,11 @@ impl GraphSettings {
             || self.run_args.param_visibility.is_hashed()
     }
 
+    /// requires dynamic lookup  
+    pub fn requires_dynamic_lookup(&self) -> bool {
+        self.num_dynamic_lookups > 0
+    }
+
     /// any kzg visibility
     pub fn module_requires_kzg(&self) -> bool {
         self.run_args.input_visibility.is_kzgcommit()
@@ -1054,7 +1067,8 @@ impl GraphCircuit {
         Ok(min_bits)
     }
 
-    fn calc_min_logrows(
+    /// calculate the minimum logrows required for the circuit
+    pub fn calc_min_logrows(
         &mut self,
         min_max_lookup: Range,
         max_range_size: i128,
@@ -1083,14 +1097,18 @@ impl GraphCircuit {
         // These are hard lower limits, we can't overflow instances or modules constraints
         let instance_logrows = self.settings().log2_total_instances();
         let module_constraint_logrows = self.settings().module_constraint_logrows();
+        let dynamic_lookup_logrows = self.settings().dynamic_lookup_logrows();
         min_logrows = std::cmp::max(
             min_logrows,
-            // max of the instance logrows and the module constraint logrows is the lower limit
-            [instance_logrows, module_constraint_logrows]
-                .iter()
-                .max()
-                .unwrap()
-                .clone(),
+            // max of the instance logrows and the module constraint logrows and the dynamic lookup logrows is the lower limit
+            *[
+                instance_logrows,
+                module_constraint_logrows,
+                dynamic_lookup_logrows,
+            ]
+            .iter()
+            .max()
+            .unwrap(),
         );
 
         // These are upper limits, going above these is wasteful, but they are not hard limits
@@ -1100,11 +1118,10 @@ impl GraphCircuit {
         max_logrows = std::cmp::min(
             max_logrows,
             // max of the model constraint logrows, min_bits, and the constants logrows is the upper limit
-            [model_constraint_logrows, min_bits, constants_logrows]
+            *[model_constraint_logrows, min_bits, constants_logrows]
                 .iter()
                 .max()
-                .unwrap()
-                .clone(),
+                .unwrap(),
         );
 
         // we now have a min and max logrows
@@ -1175,19 +1192,11 @@ impl GraphCircuit {
         settings.run_args.logrows = k;
         settings.required_range_checks = vec![(0, max_range_size)];
         let mut cs = ConstraintSystem::default();
-        // fetch gag
-        #[cfg(unix)]
-        let _r = match gag::Gag::stdout() {
-            Ok(r) => Some(r),
-            Err(_) => None,
-        };
         Self::configure_with_params(&mut cs, settings);
         #[cfg(feature = "mv-lookup")]
         let cs = cs.chunk_lookups();
         // quotient_poly_degree * params.n - 1 is the degree of the quotient polynomial
         let max_degree = cs.degree();
-        #[cfg(unix)]
-        std::mem::drop(_r);
         let quotient_poly_degree = (max_degree - 1) as u64;
         // n = 2^k
         let n = 1u64 << k;
@@ -1202,23 +1211,6 @@ impl GraphCircuit {
         true
     }
 
-    /// Calibrate the circuit to the supplied data.
-    pub fn calibrate_from_min_max(
-        &mut self,
-        min_max_lookup: Range,
-        max_range_size: i128,
-        max_logrows: Option<u32>,
-        lookup_safety_margin: i128,
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        self.calc_min_logrows(
-            min_max_lookup,
-            max_range_size,
-            max_logrows,
-            lookup_safety_margin,
-        )?;
-        Ok(())
-    }
-
     /// Runs the forward pass of the model / graph of computations and any associated hashing.
     pub fn forward(
         &self,
@@ -1518,34 +1510,18 @@ impl Circuit<Fp> for GraphCircuit {
             params.run_args.logrows as usize,
         );
 
-        let mut vars = ModelVars::new(
-            cs,
-            params.run_args.logrows as usize,
-            params.total_assignments,
-            params.run_args.num_inner_cols,
-            params.total_const_size,
-            params.module_requires_fixed(),
-        );
+        let mut vars = ModelVars::new(cs, &params);
 
         module_configs.configure_complex_modules(cs, visibility, params.module_sizes.clone());
 
         vars.instantiate_instance(
             cs,
-            params.model_instance_shapes,
+            params.model_instance_shapes.clone(),
             params.run_args.input_scale,
             module_configs.instance,
         );
 
-        let base = Model::configure(
-            cs,
-            &vars,
-            params.run_args.lookup_range,
-            params.run_args.logrows as usize,
-            params.required_lookups,
-            params.required_range_checks,
-            params.check_mode,
-        )
-        .unwrap();
+        let base = Model::configure(cs, &vars, &params).unwrap();
 
         let model_config = ModelConfig { base, vars };
 

src/graph/model.rs

@@ -99,6 +99,10 @@ pub type NodeGraph = BTreeMap<usize, NodeType>;
 pub struct DummyPassRes {
     /// number of rows use
     pub num_rows: usize,
+    /// num dynamic lookups
+    pub num_dynamic_lookups: usize,
+    /// dynamic lookup col size
+    pub dynamic_lookup_col_coord: usize,
     /// linear coordinate
     pub linear_coord: usize,
     /// total const size
@@ -540,6 +544,8 @@ impl Model {
             required_range_checks: res.range_checks.into_iter().collect(),
             model_output_scales: self.graph.get_output_scales()?,
             model_input_scales: self.graph.get_input_scales(),
+            num_dynamic_lookups: res.num_dynamic_lookups,
+            total_dynamic_col_size: res.dynamic_lookup_col_coord,
             total_const_size: res.total_const_size,
             check_mode,
             version: env!("CARGO_PKG_VERSION").to_string(),
@@ -1003,24 +1009,25 @@ impl Model {
     /// # Arguments
     /// * `meta` - The constraint system.
     /// * `vars` - The variables for the circuit.
-    /// * `run_args` - [RunArgs]
-    /// * `required_lookups` - The required lookup operations for the circuit.
+    /// * `settings` - [GraphSettings]
     pub fn configure(
         meta: &mut ConstraintSystem<Fp>,
         vars: &ModelVars<Fp>,
-        lookup_range: Range,
-        logrows: usize,
-        required_lookups: Vec<LookupOp>,
-        required_range_checks: Vec<Range>,
-        check_mode: CheckMode,
+        settings: &GraphSettings,
     ) -> Result<PolyConfig<Fp>, Box<dyn Error>> {
-        info!("configuring model");
+        debug!("configuring model");
+
+        let lookup_range = settings.run_args.lookup_range;
+        let logrows = settings.run_args.logrows as usize;
+        let num_dynamic_lookups = settings.num_dynamic_lookups;
+        let required_lookups = settings.required_lookups.clone();
+        let required_range_checks = settings.required_range_checks.clone();
 
         let mut base_gate = PolyConfig::configure(
             meta,
             vars.advices[0..2].try_into()?,
             &vars.advices[2],
-            check_mode,
+            settings.check_mode,
         );
         // set scale for HybridOp::RangeCheck and call self.conf_lookup on that op for percentage tolerance case
         let input = &vars.advices[0];
@@ -1034,6 +1041,14 @@ impl Model {
             base_gate.configure_range_check(meta, input, index, range, logrows)?;
         }
 
+        for _ in 0..num_dynamic_lookups {
+            base_gate.configure_dynamic_lookup(
+                meta,
+                vars.advices[0..2].try_into()?,
+                vars.advices[3..5].try_into()?,
+            )?;
+        }
+
         Ok(base_gate)
     }
 
@@ -1439,6 +1454,8 @@ impl Model {
             max_lookup_inputs: region.max_lookup_inputs(),
             min_lookup_inputs: region.min_lookup_inputs(),
             max_range_size: region.max_range_size(),
+            num_dynamic_lookups: region.dynamic_lookup_index(),
+            dynamic_lookup_col_coord: region.dynamic_lookup_col_coord(),
             outputs,
         };
 

src/graph/vars.rs

@@ -420,20 +420,31 @@ impl<F: PrimeField + TensorType + PartialOrd> ModelVars<F> {
     }
 
     /// Allocate all columns that will be assigned to by a model.
-    pub fn new(
-        cs: &mut ConstraintSystem<F>,
-        logrows: usize,
-        var_len: usize,
-        num_inner_cols: usize,
-        num_constants: usize,
-        module_requires_fixed: bool,
-    ) -> Self {
+    pub fn new(cs: &mut ConstraintSystem<F>, params: &GraphSettings) -> Self {
         debug!("number of blinding factors: {}", cs.blinding_factors());
 
-        let advices = (0..3)
+        let logrows = params.run_args.logrows as usize;
+        let var_len = params.total_assignments;
+        let num_inner_cols = params.run_args.num_inner_cols;
+        let num_constants = params.total_const_size;
+        let module_requires_fixed = params.module_requires_fixed();
+        let requires_dynamic_lookup = params.requires_dynamic_lookup();
+        let dynamic_lookup_size = params.total_dynamic_col_size;
+
+        let mut advices = (0..3)
             .map(|_| VarTensor::new_advice(cs, logrows, num_inner_cols, var_len))
             .collect_vec();
 
+        if requires_dynamic_lookup {
+            for _ in 0..2 {
+                let dynamic_lookup = VarTensor::new_advice(cs, logrows, 1, dynamic_lookup_size);
+                if dynamic_lookup.num_blocks() > 1 {
+                    panic!("dynamic lookup should only have one block");
+                };
+                advices.push(dynamic_lookup);
+            }
+        }
+
         debug!(
             "model uses {} advice blocks (size={})",
             advices.iter().map(|v| v.num_blocks()).sum::<usize>(),

src/lib.rs

@@ -180,10 +180,8 @@ impl RunArgs {
         if self.num_inner_cols < 1 {
             return Err("num_inner_cols must be >= 1".into());
         }
-        if self.tolerance.val > 0.0 {
-            if self.output_visibility != Visibility::Public {
-                return Err("tolerance > 0.0 requires output_visibility to be public".into());
-            }
+        if self.tolerance.val > 0.0 && self.output_visibility != Visibility::Public {
+            return Err("tolerance > 0.0 requires output_visibility to be public".into());
         }
         Ok(())
     }

src/wasm.rs

@@ -78,6 +78,17 @@ pub fn feltToBigEndian(array: wasm_bindgen::Clamped<Vec<u8>>) -> Result<String,
     Ok(format!("{:?}", felt))
 }
 
+/// Converts a felt to a little endian string
+#[wasm_bindgen]
+#[allow(non_snake_case)]
+pub fn feltToLittleEndian(array: wasm_bindgen::Clamped<Vec<u8>>) -> Result<String, JsError> {
+    let felt: Fr = serde_json::from_slice(&array[..])
+        .map_err(|e| JsError::new(&format!("Failed to deserialize field element: {}", e)))?;
+    let repr = serde_json::to_string(&felt).unwrap();
+    let b: String = serde_json::from_str(&repr).unwrap();
+    Ok(b)
+}
+
 /// Converts a hex string to a byte array
 #[wasm_bindgen]
 #[allow(non_snake_case)]

tests/integration_tests.rs

@@ -1507,7 +1507,7 @@ mod native_tests {
         if any_output_scales_smol {
             // set the tolerance to 0.0
             settings.run_args.tolerance = Tolerance {
-                val: 0.0.into(),
+                val: 0.0,
                 scale: 0.0.into(),
             };
             settings

tests/wasm.rs

@@ -9,8 +9,8 @@ mod wasm32 {
     use ezkl::pfsys;
     use ezkl::wasm::{
         bufferToVecOfFelt, compiledCircuitValidation, encodeVerifierCalldata, feltToBigEndian,
-        feltToFloat, feltToInt, genPk, genVk, genWitness, inputValidation, pkValidation,
-        poseidonHash, proofValidation, prove, settingsValidation, srsValidation,
+        feltToFloat, feltToInt, feltToLittleEndian, genPk, genVk, genWitness, inputValidation,
+        pkValidation, poseidonHash, proofValidation, prove, settingsValidation, srsValidation,
         u8_array_to_u128_le, verify, vkValidation, witnessValidation,
     };
     use halo2_solidity_verifier::encode_calldata;
@@ -89,9 +89,16 @@ mod wasm32 {
                     .unwrap();
             assert_eq!(integer, i as i128);
 
-            let hex_string = format!("{:?}", field_element);
-            let returned_string: String = feltToBigEndian(clamped).map_err(|_| "failed").unwrap();
+            let hex_string = format!("{:?}", field_element.clone());
+            let returned_string: String = feltToBigEndian(clamped.clone())
+                .map_err(|_| "failed")
+                .unwrap();
             assert_eq!(hex_string, returned_string);
+            let repr = serde_json::to_string(&field_element).unwrap();
+            let little_endian_string: String = serde_json::from_str(&repr).unwrap();
+            let returned_string: String =
+                feltToLittleEndian(clamped).map_err(|_| "failed").unwrap();
+            assert_eq!(little_endian_string, returned_string);
         }
     }
 

tests/wasm/settings.json

@@ -27,6 +27,8 @@
         "check_mode": "UNSAFE"
     },
     "num_rows": 16,
+    "total_dynamic_col_size": 0,
+    "num_dynamic_lookups": 0,
     "total_assignments": 32,
     "total_const_size": 8,
     "model_instance_shapes": [

tests/wasm/testWasm.test.ts

@@ -38,7 +38,10 @@ describe('Generate witness, prove and verify', () => {
         let pk = await readEzklArtifactsFile(path, example, 'key.pk');
         let circuit_ser = await readEzklArtifactsFile(path, example, 'network.compiled');
         circuit_settings_ser = await readEzklArtifactsFile(path, example, 'settings.json');
-        params_ser = await readEzklSrsFile(path, example);
+        // get the log rows from the circuit settings
+        const circuit_settings = deserialize(circuit_settings_ser) as any;
+        const logrows = circuit_settings.run_args.logrows as string;
+        params_ser = await readEzklSrsFile(logrows);
         const startTimeProve = Date.now();
         result = wasmFunctions.prove(witness, pk, circuit_ser, params_ser);
         const endTimeProve = Date.now();
@@ -54,6 +57,7 @@ describe('Generate witness, prove and verify', () => {
         let result
         const vk = await readEzklArtifactsFile(path, example, 'key.vk');
         const startTimeVerify = Date.now();
+        params_ser = await readEzklSrsFile("1");
         result = wasmFunctions.verify(proof_ser, vk, circuit_settings_ser, params_ser);
         const result_ref = wasmFunctions.verify(proof_ser_ref, vk, circuit_settings_ser, params_ser);
         const endTimeVerify = Date.now();

tests/wasm/utils.ts

@@ -16,15 +16,7 @@ export async function readEzklArtifactsFile(path: string, example: string, filen
   return new Uint8ClampedArray(buffer.buffer);
 }
 
-export async function readEzklSrsFile(path: string, example: string): Promise<Uint8ClampedArray> {
-  // const settingsPath = path.join(__dirname, '..', '..', 'ezkl', 'examples', 'onnx', example, 'settings.json');
-
-  const settingsPath = `${path}/${example}/settings.json`
-  const settingsBuffer = await fs.readFile(settingsPath, { encoding: 'utf-8' });
-  const settings = JSONBig.parse(settingsBuffer);
-  const logrows = settings.run_args.logrows;
-  // const filePath = path.join(__dirname, '..', '..', 'ezkl', 'examples', 'onnx', `kzg${logrows}.srs`);
-  // srs path is at $HOME/.ezkl/srs
+export async function readEzklSrsFile(logrows: string): Promise<Uint8ClampedArray> {
   const filePath = `${userHomeDir}/.ezkl/srs/kzg${logrows}.srs`
   const buffer = await fs.readFile(filePath);
   return new Uint8ClampedArray(buffer.buffer);