fix: lookup safety = 1 during calibration falls OOR (#750)

.github/workflows/rust.yml

@@ -392,6 +392,8 @@ jobs:
       - name: Replace memory definition in nodejs
         run: |
           sed -i "3s|.*|imports['env'] = {memory: new WebAssembly.Memory({initial:20,maximum:65536,shared:true})}|" tests/wasm/nodejs/ezkl.js
+      - name: KZG prove and verify tests (public outputs)
+        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_tight_lookup_::t
       - name: IPA prove and verify tests
         run: cargo nextest run --release --verbose tests::ipa_prove_and_verify_::t --test-threads 1
       - name: IPA prove and verify tests (ipa outputs)
@@ -412,8 +414,6 @@ jobs:
         run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_with_overflow_fixed_params_
       - name: KZG prove and verify tests (public outputs)
         run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t
-      - name: KZG prove and verify tests (public outputs + column overflow)
-        run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t
       - name: KZG prove and verify tests (public inputs)
         run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_public_input
       - name: KZG prove and verify tests (fixed params)
@@ -456,7 +456,6 @@ jobs:
       - name: KZG prove and verify tests (hashed outputs)
         run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_hashed --features icicle  --test-threads 1
 
-
   prove-and-verify-mock-aggr-tests:
     runs-on: self-hosted
     needs: [build, library-tests]
@@ -509,8 +508,6 @@ jobs:
       - name: KZG tests
         run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_prove_and_verify_  --test-threads 4 -- --include-ignored
 
-
-
   prove-and-verify-aggr-evm-tests:
     runs-on: large-self-hosted
     needs: [build, library-tests]

examples/notebooks/ezkl_demo.ipynb

@@ -696,10 +696,12 @@
         "for i, value in enumerate(proof[\"instances\"]):\n",
         "    for j, field_element in enumerate(value):\n",
         "        onchain_input_array.append(ezkl.felt_to_big_endian(field_element))\n",
-        "        formatted_output += str(onchain_input_array[-1])\n",
+        "        formatted_output += '\"' + str(onchain_input_array[-1]) + '\"'\n",
         "        if j != len(value) - 1:\n",
         "            formatted_output += \", \"\n",
-        "    formatted_output += \"]\"\n",
+        "    if i != len(proof[\"instances\"]) - 1:\n",
+        "        formatted_output += \", \"\n",
+        "formatted_output += \"]\"\n",
         "\n",
         "# This will be the values you use onchain\n",
         "# copy them over to remix and see if they verify\n",

src/graph/mod.rs

@@ -503,7 +503,9 @@ impl GraphSettings {
     }
 
     fn constants_logrows(&self) -> u32 {
-        (self.total_const_size as f64).log2().ceil() as u32
+        (self.total_const_size as f64 / self.run_args.num_inner_cols as f64)
+            .log2()
+            .ceil() as u32
     }
 
     /// calculate the total number of instances
@@ -1049,14 +1051,10 @@ impl GraphCircuit {
     }
 
     fn calc_safe_lookup_range(min_max_lookup: Range, lookup_safety_margin: i128) -> Range {
-        let mut margin = (
+        let margin = (
             lookup_safety_margin * min_max_lookup.0,
             lookup_safety_margin * min_max_lookup.1,
         );
-        if lookup_safety_margin == 1 {
-            margin.0 += 4;
-            margin.1 += 4;
-        }
 
         margin
     }

tests/integration_tests.rs

@@ -790,7 +790,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -799,7 +799,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 3, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 3, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -808,7 +808,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 4, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 4, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -817,7 +817,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 8, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 8, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -826,7 +826,16 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 2);
+               test_dir.close().unwrap();
+            }
+
+            #(#[test_case(TESTS[N])])*
+            fn kzg_prove_and_verify_tight_lookup_(test: &str) {
+                crate::native_tests::init_binary();
+                let test_dir = TempDir::new(test).unwrap();
+                let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 1);
                test_dir.close().unwrap();
             }
 
@@ -835,7 +844,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::IPA);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::IPA, 2);
                test_dir.close().unwrap();
             }
 
@@ -844,7 +853,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "public", "private", "public", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "public", "private", "public", 1, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -853,7 +862,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -862,7 +871,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "hashed", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "hashed", 1, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -871,7 +880,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::KZG);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::KZG, 2);
                test_dir.close().unwrap();
             }
 
@@ -880,7 +889,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-               prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::IPA);
+               prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::IPA, 2);
                test_dir.close().unwrap();
             }
 
@@ -895,7 +904,7 @@ mod native_tests {
                     let test_dir = TempDir::new(test).unwrap();
                     env_logger::init();
                     let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-                    prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, true, "single", Commitments::KZG);
+                    prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, true, "single", Commitments::KZG, 2);
                     #[cfg(not(feature = "icicle"))]
                     run_js_tests(path, test.to_string(), "testWasm", false);
                     // test_dir.close().unwrap();
@@ -908,7 +917,7 @@ mod native_tests {
                     let test_dir = TempDir::new(test).unwrap();
                     env_logger::init();
                     let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-                    prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, true, "single", Commitments::KZG);
+                    prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, true, "single", Commitments::KZG, 2);
                     #[cfg(not(feature = "icicle"))]
                     run_js_tests(path, test.to_string(), "testWasm", false);
                     test_dir.close().unwrap();
@@ -924,7 +933,7 @@ mod native_tests {
                 crate::native_tests::init_binary();
                 let test_dir = TempDir::new(test).unwrap();
                 let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test);
-                prove_and_verify(path, test.to_string(), "unsafe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG);
+                prove_and_verify(path, test.to_string(), "unsafe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG, 2);
                 test_dir.close().unwrap();
             }
 
@@ -1338,6 +1347,7 @@ mod native_tests {
             false,
             &mut tolerance,
             Commitments::KZG,
+            2,
         );
 
         if tolerance > 0.0 {
@@ -1478,6 +1488,7 @@ mod native_tests {
         div_rebasing: bool,
         tolerance: &mut f32,
         commitment: Commitments,
+        lookup_safety_margin: usize,
     ) {
         let mut args = vec![
             "gen-settings".to_string(),
@@ -1518,6 +1529,7 @@ mod native_tests {
                 test_dir, example_name
             ),
             format!("--target={}", cal_target),
+            format!("--lookup-safety-margin={}", lookup_safety_margin),
         ];
 
         if let Some(scales) = scales_to_use {
@@ -1613,6 +1625,7 @@ mod native_tests {
             div_rebasing,
             &mut 0.0,
             Commitments::KZG,
+            2,
         );
 
         println!(
@@ -1666,6 +1679,7 @@ mod native_tests {
             false,
             "for-aggr",
             Commitments::KZG,
+            2,
         );
         let status = Command::new(format!("{}/release/ezkl", *CARGO_TARGET_DIR))
             .args([
@@ -1700,6 +1714,7 @@ mod native_tests {
             false,
             "for-aggr",
             Commitments::KZG,
+            2,
         );
 
         download_srs(23, commitment);
@@ -1861,6 +1876,7 @@ mod native_tests {
         overflow: bool,
         proof_type: &str,
         commitment: Commitments,
+        lookup_safety_margin: usize,
     ) {
         let target_str = if overflow {
             "resources/col-overflow"
@@ -1881,6 +1897,7 @@ mod native_tests {
             false,
             &mut 0.0,
             commitment,
+            lookup_safety_margin,
         );
 
         let settings_path = format!("{}/{}/settings.json", test_dir, example_name);
@@ -1993,6 +2010,7 @@ mod native_tests {
             false,
             "single",
             Commitments::KZG,
+            2,
         );
 
         let settings_path = format!("{}/{}/settings.json", test_dir, example_name);
@@ -2087,6 +2105,7 @@ mod native_tests {
             false,
             "single",
             Commitments::KZG,
+            2,
         );
 
         let settings_path = format!("{}/{}/settings.json", test_dir, example_name);
@@ -2235,6 +2254,7 @@ mod native_tests {
             false,
             &mut 0.0,
             Commitments::KZG,
+            2,
         );
 
         let model_path = format!("{}/{}/network.compiled", test_dir, example_name);