fix: calibration should only consider local scales (#691)

---------

Co-authored-by: dante <45801863+alexander-camuto@users.noreply.github.com>

.github/workflows/rust.yml

@@ -643,10 +643,10 @@ jobs:
       #     # now dump the contents of the file into a file called kaggle.json
       #     echo $KAGGLE_API_KEY > /home/ubuntu/.kaggle/kaggle.json
       #     chmod 600 /home/ubuntu/.kaggle/kaggle.json
-      - name: NBEATS tutorial
-        run: source .env/bin/activate; cargo nextest run py_tests::tests::nbeats_
       - name: Voice tutorial
         run: source .env/bin/activate; cargo nextest run py_tests::tests::voice_
+      - name: NBEATS tutorial
+        run: source .env/bin/activate; cargo nextest run py_tests::tests::nbeats_
       - name: All notebooks
         run: source .env/bin/activate; cargo nextest run py_tests::tests::run_notebook_ --test-threads 1
       - name: Tictactoe tutorials

examples/notebooks/data_attest.ipynb

@@ -271,7 +271,7 @@
                 "The graph input for on chain data sources is formatted completely differently compared to file based data sources.\n",
                 "\n",
                 "- For file data sources, the raw floating point values that eventually get quantized, converted into field elements and stored in `witness.json` to be consumed by the circuit are stored. The output data contains the expected floating point values returned as outputs from running your vanilla pytorch model on the given inputs.\n",
-                "- For on chain data sources, the input_data field contains all the data necessary to read and format the on chain data into something digestable by EZKL (aka field elemenets :-D). \n",
+                "- For on chain data sources, the input_data field contains all the data necessary to read and format the on chain data into something digestable by EZKL (aka field elements :-D). \n",
                 "Here is what the schema for an on-chain data source graph input file should look like:\n",
                 "    \n",
                 "```json\n",

examples/notebooks/kzg_vis.ipynb

@@ -7,7 +7,7 @@
             "source": [
                 "# kzg-ezkl\n",
                 "\n",
-                "Here's an example leveraging EZKL whereby the inputs to the model, and the model params themselves, are commited to using kzg-commitments inside a circuit.\n",
+                "Here's an example leveraging EZKL whereby the inputs to the model, and the model params themselves, are committed to using kzg-commitments inside a circuit.\n",
                 "\n",
                 "In this setup:\n",
                 "- the commitments are publicly known to the prover and verifier\n",
@@ -166,7 +166,7 @@
                 "Shoutouts: \n",
                 "\n",
                 "- [summa-solvency](https://github.com/summa-dev/summa-solvency) for their help with the poseidon hashing chip. \n",
-                "- [timeofey](https://github.com/timoftime) for providing inspiration in our developement of the el-gamal encryption circuit in Halo2. "
+                "- [timeofey](https://github.com/timoftime) for providing inspiration in our development of the el-gamal encryption circuit in Halo2. "
             ]
         },
         {

examples/notebooks/simple_demo_public_network_output.ipynb

@@ -8,7 +8,7 @@
             "source": [
                 "## EZKL Jupyter Notebook Demo \n",
                 "\n",
-                "Here we demonstrate how to use the EZKL package to run a publicly known / committted to network on some private data, producing a public output.\n"
+                "Here we demonstrate how to use the EZKL package to run a publicly known / committed to network on some private data, producing a public output.\n"
             ]
         },
         {

examples/notebooks/tictactoe_binary_classification.ipynb

@@ -154,7 +154,7 @@
             "source": [
                 "## Create a neural net to verify the execution of the tic tac toe model\n",
                 "\n",
-                "1. Given the data generated above classify whether the tic tac toe games are valid. This approach uses a binary classification as the tic tac toe state space is fairly small. For larger state spaces we will want to use anomaly detection based approachs"
+                "1. Given the data generated above classify whether the tic tac toe games are valid. This approach uses a binary classification as the tic tac toe state space is fairly small. For larger state spaces, we will want to use anomaly detection based approaches."
             ]
         },
         {

examples/notebooks/world_rotation.ipynb

@@ -49,7 +49,7 @@
     "import torch\n",
     "import math\n",
     "\n",
-    "# these are constatns for the rotation\n",
+    "# these are constants for the rotation\n",
     "phi = torch.tensor(5 * math.pi / 180)\n",
     "s = torch.sin(phi)\n",
     "c = torch.cos(phi)\n",

src/execute.rs

@@ -834,7 +834,6 @@ pub(crate) fn calibrate(
             Ok(r) => Some(r),
             Err(_) => None,
         };
-
         let key = (input_scale, param_scale, scale_rebase_multiplier);
         forward_pass_res.insert(key, vec![]);
 
@@ -847,7 +846,15 @@ pub(crate) fn calibrate(
 
         let mut circuit = match GraphCircuit::from_run_args(&local_run_args, &model_path) {
             Ok(c) => c,
-            Err(_) => return Err(format!("failed to create circuit from run args").into()),
+            Err(e) => {
+                // drop the gag
+                #[cfg(unix)]
+                std::mem::drop(_r);
+                #[cfg(unix)]
+                std::mem::drop(_q);
+                debug!("circuit creation from run args failed: {:?}", e);
+                continue;
+            }
         };
 
         chunks
@@ -874,16 +881,18 @@ pub(crate) fn calibrate(
             .collect::<Result<Vec<()>, String>>()?;
 
         let min_lookup_range = forward_pass_res
+            .get(&key)
+            .unwrap()
             .iter()
-            .map(|x| x.1.iter().map(|x| x.min_lookup_inputs))
-            .flatten()
+            .map(|x| x.min_lookup_inputs)
             .min()
             .unwrap_or(0);
 
         let max_lookup_range = forward_pass_res
+            .get(&key)
+            .unwrap()
             .iter()
-            .map(|x| x.1.iter().map(|x| x.max_lookup_inputs))
-            .flatten()
+            .map(|x| x.max_lookup_inputs)
             .max()
             .unwrap_or(0);
 
@@ -930,7 +939,7 @@ pub(crate) fn calibrate(
                 found_settings.as_json()?.to_colored_json_auto()?
             );
         } else {
-            debug!("calibration failed");
+            debug!("calibration failed {}", res.err().unwrap());
         }
 
         pb.inc(1);