refactor: lookup-less layer norm (#706)

.github/workflows/rust.yml

@@ -198,8 +198,6 @@ jobs:
         #   chromedriver-version: "115.0.5790.102"
       - name: Install wasm32-unknown-unknown
         run: rustup target add wasm32-unknown-unknown
-      - name: Install wasm runner
-        run: cargo install wasm-server-runner
       - name: Add rust-src
         run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
       - name: Run wasm verifier tests
@@ -352,9 +350,6 @@ jobs:
       - name: Add wasm32-unknown-unknown target
         run: rustup target add wasm32-unknown-unknown
 
-      - name: Install wasm-server-runner
-        run: cargo install wasm-server-runner
-
       - name: Add rust-src
         run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
       - uses: actions/checkout@v3

.github/workflows/wasm.yml

@@ -29,9 +29,6 @@ jobs:
       - name: Add wasm32-unknown-unknown target
         run: rustup target add wasm32-unknown-unknown
 
-      - name: Install wasm-server-runner
-        run: cargo install wasm-server-runner
-
       - name: Add rust-src
         run: rustup component add rust-src --toolchain nightly-2023-08-24-x86_64-unknown-linux-gnu
       - name: Install binaryen

src/circuit/ops/base.rs

@@ -125,8 +125,8 @@ impl BaseOp {
             BaseOp::Sum => 1,
             BaseOp::SumInit => 1,
             BaseOp::Range { .. } => 1,
-            BaseOp::IsZero => 1,
-            BaseOp::IsBoolean => 1,
+            BaseOp::IsZero => 0,
+            BaseOp::IsBoolean => 0,
         }
     }
 

src/circuit/ops/chip.rs

@@ -276,9 +276,20 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
 
                 let constraints = match base_op {
                     BaseOp::IsBoolean => {
-                        vec![(qis[1].clone()) * (qis[1].clone() - Expression::Constant(F::from(1)))]
+                        let expected_output: Tensor<Expression<F>> = output
+                            .query_rng(meta, *block_idx, *inner_col_idx, 0, 1)
+                            .expect("non accum: output query failed");
+
+                        let output = expected_output[base_op.constraint_idx()].clone();
+
+                        vec![(output.clone()) * (output.clone() - Expression::Constant(F::from(1)))]
+                    }
+                    BaseOp::IsZero => {
+                        let expected_output: Tensor<Expression<F>> = output
+                            .query_rng(meta, *block_idx, *inner_col_idx, 0, 1)
+                            .expect("non accum: output query failed");
+                        vec![expected_output[base_op.constraint_idx()].clone()]
                     }
-                    BaseOp::IsZero => vec![qis[1].clone()],
                     _ => {
                         let expected_output: Tensor<Expression<F>> = output
                             .query_rng(meta, *block_idx, *inner_col_idx, rotation_offset, rng)
@@ -512,10 +523,10 @@ impl<F: PrimeField + TensorType + PartialOrd> BaseConfig<F> {
 
         // we borrow mutably twice so we need to do this dance
 
-        let range_check = if !self.range_checks.contains_key(&range) {
+        let range_check = if let std::collections::btree_map::Entry::Vacant(e) = self.range_checks.entry(range) {
             // as all tables have the same input we see if there's another table who's input we can reuse
             let range_check = RangeCheck::<F>::configure(cs, range);
-            self.range_checks.insert(range, range_check.clone());
+            e.insert(range_check.clone());
             range_check
         } else {
             return Ok(());

src/circuit/ops/hybrid.rs

@@ -160,23 +160,17 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for HybridOp {
             }
             HybridOp::ReduceArgMax { dim } => {
                 let res = tensor::ops::argmax_axes(&x, *dim)?;
-                let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                let mut inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
                 let inter =
                     Op::f(&HybridOp::ReduceMax { axes: vec![*dim] }, inputs)?.intermediate_lookups;
-                inter_equals.extend(inter);
 
-                (res.clone(), inter_equals)
+                (res, inter)
             }
             HybridOp::ReduceArgMin { dim } => {
                 let res = tensor::ops::argmin_axes(&x, *dim)?;
-                let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                let mut inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
                 let inter =
                     Op::f(&HybridOp::ReduceMin { axes: vec![*dim] }, inputs)?.intermediate_lookups;
-                inter_equals.extend(inter);
 
-                (res.clone(), inter_equals)
+                (res, inter)
             }
             HybridOp::Gather { dim, constant_idx } => {
                 if let Some(idx) = constant_idx {
@@ -185,18 +179,14 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for HybridOp {
                     (res.clone(), vec![])
                 } else {
                     let y = inputs[1].clone().map(|x| felt_to_i128(x));
-                    let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                    let inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
                     let res = tensor::ops::gather(&x, &y.map(|x| x as usize), *dim)?;
-                    (res.clone(), inter_equals)
+                    (res.clone(), vec![])
                 }
             }
-            HybridOp::OneHot { dim, num_classes } => {
-                let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                let inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
-                let res = tensor::ops::one_hot(&x, *num_classes, *dim)?;
-                (res.clone(), inter_equals)
-            }
+            HybridOp::OneHot { dim, num_classes } => (
+                tensor::ops::one_hot(&x, *num_classes, *dim)?.clone(),
+                vec![],
+            ),
             HybridOp::TopK { dim, k, largest } => {
                 let res = tensor::ops::topk_axes(&x, *k, *dim, *largest)?;
 
@@ -228,10 +218,8 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for HybridOp {
                     (res.clone(), vec![])
                 } else {
                     let y = inputs[1].clone().map(|x| felt_to_i128(x));
-                    let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                    let inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
                     let res = tensor::ops::gather_elements(&x, &y.map(|x| x as usize), *dim)?;
-                    (res.clone(), inter_equals)
+                    (res.clone(), vec![])
                 }
             }
             HybridOp::ScatterElements { dim, constant_idx } => {
@@ -243,10 +231,8 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for HybridOp {
                 } else {
                     let idx = inputs[1].clone().map(|x| felt_to_i128(x) as usize);
                     let src = inputs[2].clone().map(|x| felt_to_i128(x));
-                    let indices = Tensor::from(0..x.dims()[*dim] as i128);
-                    let inter_equals: Vec<Tensor<i128>> = vec![indices.clone(), -indices];
                     let res = tensor::ops::scatter(&x, &idx, &src, *dim)?;
-                    (res.clone(), inter_equals)
+                    (res.clone(), vec![])
                 }
             }
             HybridOp::MaxPool2d {
@@ -441,7 +427,7 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for HybridOp {
                         region,
                         values.try_into()?,
                         &LookupOp::Div {
-                            denom: denom.clone(),
+                            denom: *denom,
                         },
                     )?
                 }

src/circuit/ops/layouts.rs

@@ -789,14 +789,7 @@ fn one_hot<F: PrimeField + TensorType + PartialOrd>(
     let assigned_input = region.assign(&config.inputs[0], &input)?;
 
     // now assert all elems are 0 or 1
-    let assigned_output = region.assign(&config.inputs[1], &output)?;
-    if !region.is_dummy() {
-        for i in 0..assigned_output.len() {
-            let (x, y, z) = config.output.cartesian_coord(region.linear_coord() + i);
-            let selector = config.selectors.get(&(BaseOp::IsBoolean, x, y));
-            region.enable(selector, z)?;
-        }
-    }
+    let assigned_output = boolean_identity(config, region, &[output.clone()], true)?;
     region.increment(std::cmp::max(assigned_output.len(), assigned_input.len()));
 
     let sum = sum(config, region, &[assigned_output.clone()])?;
@@ -1701,10 +1694,28 @@ pub fn equals<F: PrimeField + TensorType + PartialOrd>(
     values: &[ValTensor<F>; 2],
 ) -> Result<ValTensor<F>, Box<dyn Error>> {
     let diff = pairwise(config, region, values, BaseOp::Sub)?;
+    let diff_inverse = diff.inverse()?;
+    let product_diff_and_invert =
+        pairwise(config, region, &[diff.clone(), diff_inverse], BaseOp::Mult)?;
 
-    let res = nonlinearity(config, region, &[diff], &LookupOp::KroneckerDelta)?;
+    // constant of 1
+    let mut ones = Tensor::from(vec![ValType::Constant(F::from(1))].into_iter());
+    ones.set_visibility(&crate::graph::Visibility::Fixed);
 
-    Ok(res)
+    // subtract
+    let output = pairwise(
+        config,
+        region,
+        &[ones.into(), product_diff_and_invert],
+        BaseOp::Sub,
+    )?;
+
+    // take the product of diff and output
+    let prod_check = pairwise(config, region, &[diff, output.clone()], BaseOp::Mult)?;
+
+    is_zero_identity(config, region, &[prod_check], false)?;
+
+    Ok(output)
 }
 
 /// Xor boolean operation
@@ -1768,21 +1779,7 @@ pub fn iff<F: PrimeField + TensorType + PartialOrd>(
             .into();
 
     // make sure mask is boolean
-    let assigned_mask = region.assign(&config.inputs[1], mask)?;
-
-    // Enable the selectors
-    if !region.is_dummy() {
-        (0..assigned_mask.len())
-            .map(|i| {
-                let (x, y, z) = config.inputs[1].cartesian_coord(region.linear_coord() + i);
-                let selector = config.selectors.get(&(BaseOp::IsBoolean, x, y));
-                region.enable(selector, z)?;
-                Ok(())
-            })
-            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
-    }
-
-    region.increment(assigned_mask.len());
+    let assigned_mask = boolean_identity(config, region, &[mask.clone()], true)?;
 
     let one_minus_mask = pairwise(config, region, &[unit, assigned_mask.clone()], BaseOp::Sub)?;
 
@@ -1880,13 +1877,11 @@ pub fn sumpool<F: PrimeField + TensorType + PartialOrd>(
     last_elem.reshape(&[&[batch_size, image_channels], shape].concat())?;
 
     if normalized {
-        last_elem = nonlinearity(
+        last_elem = div(
             config,
             region,
             &[last_elem],
-            &LookupOp::Div {
-                denom: utils::F32((kernel_shape.0 * kernel_shape.1) as f32),
-            },
+            F::from((kernel_shape.0 * kernel_shape.1) as u64),
         )?;
     }
     Ok(last_elem)
@@ -2383,18 +2378,60 @@ pub fn identity<F: PrimeField + TensorType + PartialOrd>(
     Ok(output)
 }
 
+/// is zero identity constraint. Usually used to constrain an instance column to an advice so the returned cells / values can be operated upon.
+pub fn is_zero_identity<F: PrimeField + TensorType + PartialOrd>(
+    config: &BaseConfig<F>,
+    region: &mut RegionCtx<F>,
+    values: &[ValTensor<F>; 1],
+    assign: bool,
+) -> Result<ValTensor<F>, Box<dyn Error>> {
+    let output = if assign || !values[0].get_const_indices()?.is_empty() {
+        let output = region.assign(&config.output, &values[0])?;
+        region.increment(output.len());
+        output
+    } else {
+        values[0].clone()
+    };
+    // Enable the selectors
+    if !region.is_dummy() {
+        (0..output.len())
+            .map(|j| {
+                let index = region.linear_coord() - j - 1;
+
+                let (x, y, z) = config.output.cartesian_coord(index);
+                let selector = config.selectors.get(&(BaseOp::IsZero, x, y));
+
+                region.enable(selector, z)?;
+                Ok(())
+            })
+            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
+    }
+
+    Ok(output)
+}
+
 /// Boolean identity constraint. Usually used to constrain an instance column to an advice so the returned cells / values can be operated upon.
 pub fn boolean_identity<F: PrimeField + TensorType + PartialOrd>(
     config: &BaseConfig<F>,
     region: &mut RegionCtx<F>,
     values: &[ValTensor<F>; 1],
+    assign: bool,
 ) -> Result<ValTensor<F>, Box<dyn Error>> {
-    let output = region.assign(&config.inputs[1], &values[0])?;
+    let output = if assign || !values[0].get_const_indices()?.is_empty() {
+        // get zero constants indices
+        let output = region.assign(&config.output, &values[0])?;
+        region.increment(output.len());
+        output
+    } else {
+        values[0].clone()
+    };
     // Enable the selectors
     if !region.is_dummy() {
         (0..output.len())
             .map(|j| {
-                let (x, y, z) = config.inputs[1].cartesian_coord(region.linear_coord() + j);
+                let index = region.linear_coord() - j - 1;
+
+                let (x, y, z) = config.output.cartesian_coord(index);
                 let selector = config.selectors.get(&(BaseOp::IsBoolean, x, y));
 
                 region.enable(selector, z)?;
@@ -2402,7 +2439,6 @@ pub fn boolean_identity<F: PrimeField + TensorType + PartialOrd>(
             })
             .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
     }
-    region.increment(output.len());
 
     Ok(output)
 }
@@ -2471,7 +2507,7 @@ pub fn range_check<F: PrimeField + TensorType + PartialOrd>(
                 let (x, y, z) = config
                     .lookup_input
                     .cartesian_coord(region.linear_coord() + i);
-                let selector = config.range_check_selectors.get(&(range.clone(), x, y));
+                let selector = config.range_check_selectors.get(&(*range, x, y));
                 region.enable(selector, z)?;
                 Ok(())
             })
@@ -2581,22 +2617,6 @@ pub fn nonlinearity<F: PrimeField + TensorType + PartialOrd>(
     Ok(output)
 }
 
-/// mean function layout
-pub fn mean<F: PrimeField + TensorType + PartialOrd>(
-    config: &BaseConfig<F>,
-    region: &mut RegionCtx<F>,
-    values: &[ValTensor<F>; 1],
-    scale: usize,
-) -> Result<ValTensor<F>, Box<dyn Error>> {
-    let x = &values[0];
-
-    let sum_x = sum(config, region, &[x.clone()])?;
-    let nl = LookupOp::Div {
-        denom: utils::F32((scale * x.len()) as f32),
-    };
-    nonlinearity(config, region, &[sum_x], &nl)
-}
-
 /// Argmax
 pub fn argmax<F: PrimeField + TensorType + PartialOrd>(
     config: &BaseConfig<F>,
@@ -2709,24 +2729,8 @@ pub fn max<F: PrimeField + TensorType + PartialOrd>(
     )?;
     // relu(x - max(x - 1))
     let relu = nonlinearity(config, region, &[diff], &LookupOp::ReLU)?;
-
-    let len = relu.dims().iter().product();
-
-    // y_i*(1 - y_i) =0 // assert the values are either 0 or 1
-    region.assign(&config.inputs[1], &relu)?;
-
-    if !region.is_dummy() {
-        (0..len)
-            .map(|i| {
-                let (x, y, z) = config.inputs[1].cartesian_coord(region.linear_coord() + i);
-                let selector = config.selectors.get(&(BaseOp::IsBoolean, x, y));
-                region.enable(selector, z)?;
-                Ok(())
-            })
-            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
-    }
-
-    region.increment(len);
+    // constraining relu(x - max(x - 1)) = 0/1
+    boolean_identity(config, region, &[relu.clone()], false)?;
 
     // sum(relu(x - max(x - 1)))
     let sum_relu = sum(config, region, &[relu])?;
@@ -2737,13 +2741,7 @@ pub fn max<F: PrimeField + TensorType + PartialOrd>(
         nonlinearity(config, region, &[one_minus_sum_relu], &LookupOp::ReLU)?;
 
     // constraining 1 - sum(relu(x - max(x - 1))) = 0
-    region.assign(&config.inputs[1], &relu_one_minus_sum_relu)?;
-
-    let (x, y, z) = config.output.cartesian_coord(region.linear_coord());
-    let selector = config.selectors.get(&(BaseOp::IsZero, x, y));
-    region.enable(selector, z)?;
-
-    region.increment(relu_one_minus_sum_relu.len());
+    is_zero_identity(config, region, &[relu_one_minus_sum_relu], false)?;
 
     Ok(assigned_max_val)
 }
@@ -2788,23 +2786,8 @@ pub fn min<F: PrimeField + TensorType + PartialOrd>(
 
     // relu(min(x + 1)  - x)
     let relu = nonlinearity(config, region, &[diff], &LookupOp::ReLU)?;
-
-    let len = relu.dims().iter().product();
-
-    region.assign(&config.inputs[1], &relu)?;
-    // y_i*(1 - y_i) =0 // assert the values are either 0 or 1
-    if !region.is_dummy() {
-        (0..len)
-            .map(|i| {
-                let (x, y, z) = config.inputs[1].cartesian_coord(region.linear_coord() + i);
-                let selector = config.selectors.get(&(BaseOp::IsBoolean, x, y));
-                region.enable(selector, z)?;
-                Ok(())
-            })
-            .collect::<Result<Vec<_>, Box<dyn Error>>>()?;
-    }
-
-    region.increment(len);
+    // constraining relu(min(x + 1) - x) = 0/1
+    boolean_identity(config, region, &[relu.clone()], false)?;
 
     // sum(relu(min(x + 1) - x))
     let sum_relu = sum(config, region, &[relu])?;
@@ -2815,14 +2798,8 @@ pub fn min<F: PrimeField + TensorType + PartialOrd>(
     let relu_one_minus_sum_relu =
         nonlinearity(config, region, &[one_minus_sum_relu], &LookupOp::ReLU)?;
 
-    region.assign(&config.inputs[1], &relu_one_minus_sum_relu)?;
-
     // constraining product to 0
-    let (x, y, z) = config.output.cartesian_coord(region.linear_coord());
-    let selector = config.selectors.get(&(BaseOp::IsZero, x, y));
-    region.enable(selector, z)?;
-
-    region.increment(relu_one_minus_sum_relu.len());
+    is_zero_identity(config, region, &[relu_one_minus_sum_relu], false)?;
 
     Ok(assigned_min_val)
 }
@@ -3041,15 +3018,8 @@ pub fn range_check_percent<F: PrimeField + TensorType + PartialOrd>(
     // Add the lower_bound and upper_bound
     let sum = pairwise(config, region, &[lower_bound, upper_bound], BaseOp::Add)?;
 
-    // Assign the sum tensor to the inputs
-    region.assign(&config.inputs[1], &sum)?;
-
     // Constrain the sum to be all zeros
-    let (x, y, z) = config.output.cartesian_coord(region.linear_coord());
-    let selector = config.selectors.get(&(BaseOp::IsZero, x, y));
-    region.enable(selector, z)?;
-
-    region.increment(sum.len());
+    is_zero_identity(config, region, &[sum.clone()], false)?;
 
     Ok(sum)
 }

src/circuit/ops/mod.rs

@@ -201,6 +201,7 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for Input {
                         config,
                         region,
                         values[..].try_into()?,
+                        true,
                     )?))
                 }
                 _ => Ok(Some(super::layouts::identity(

src/execute.rs

@@ -726,11 +726,11 @@ impl AccuracyResults {
             let percentage_error = error.enum_map(|i, x| Ok::<_, TensorError>(x / original[i]))?;
             let abs_percentage_error = percentage_error.map(|x| x.abs());
 
-            errors.extend(error.into_iter());
-            abs_errors.extend(abs_error.into_iter());
-            squared_errors.extend(squared_error.into_iter());
-            percentage_errors.extend(percentage_error.into_iter());
-            abs_percentage_errors.extend(abs_percentage_error.into_iter());
+            errors.extend(error);
+            abs_errors.extend(abs_error);
+            squared_errors.extend(squared_error);
+            percentage_errors.extend(percentage_error);
+            abs_percentage_errors.extend(abs_percentage_error);
         }
 
         let mean_percent_error =

src/graph/model.rs

@@ -1173,8 +1173,8 @@ impl Model {
         );
         // set scale for HybridOp::RangeCheck and call self.conf_lookup on that op for percentage tolerance case
         let input = &vars.advices[0];
-        let output = &vars.advices[1];
-        let index = &vars.advices[2];
+        let output = &vars.advices[2];
+        let index = &vars.advices[1];
         for op in required_lookups {
             base_gate.configure_lookup(meta, input, output, index, lookup_range, logrows, &op)?;
         }

src/graph/node.rs

@@ -148,7 +148,7 @@ impl RebaseScale {
                 SupportedOp::RebaseScale(RebaseScale {
                     inner: op.inner.clone(),
                     target_scale: op.target_scale,
-                    multiplier: multiplier,
+                    multiplier,
                     rebase_op: HybridOp::Div {
                         denom: crate::circuit::utils::F32((multiplier) as f32),
                         use_range_check_for_int: !div_rebasing,

src/tensor/val.rs

@@ -871,3 +871,30 @@ impl<F: PrimeField + TensorType + PartialOrd> ValTensor<F> {
         }
     }
 }
+
+impl<F: PrimeField + TensorType + PartialOrd> ValTensor<F> {
+    /// inverts the inner values
+    pub fn inverse(&self) -> Result<ValTensor<F>, Box<dyn Error>> {
+        let mut cloned_self = self.clone();
+
+        match &mut cloned_self {
+            ValTensor::Value {
+                inner: v, dims: d, ..
+            } => {
+                *v = v.map(|x| match x {
+                    ValType::AssignedValue(v) => ValType::AssignedValue(v.invert()),
+                    ValType::PrevAssigned(v) | ValType::AssignedConstant(v, ..) => {
+                        ValType::AssignedValue(v.value_field().invert())
+                    }
+                    ValType::Value(v) => ValType::Value(v.map(|x| x.invert().unwrap_or(F::ZERO))),
+                    ValType::Constant(v) => ValType::Constant(v.invert().unwrap_or(F::ZERO)),
+                });
+                *d = v.dims().to_vec();
+            }
+            ValTensor::Instance { .. } => {
+                return Err(Box::new(TensorError::WrongMethod));
+            }
+        };
+        Ok(cloned_self)
+    }
+}

tests/integration_tests.rs

@@ -192,86 +192,86 @@ mod native_tests {
     ];
 
     const TESTS: [&str; 77] = [
-        "1l_mlp",
+        "1l_mlp", //0
         "1l_slice",
         "1l_concat",
         "1l_flatten",
         // "1l_average",
         "1l_div",
-        "1l_pad",
+        "1l_pad", // 5
         "1l_reshape",
         "1l_eltwise_div",
         "1l_sigmoid",
         "1l_sqrt",
-        "1l_softmax",
+        "1l_softmax", //10
         // "1l_instance_norm",
         "1l_batch_norm",
         "1l_prelu",
         "1l_leakyrelu",
         "1l_gelu_noappx",
         // "1l_gelu_tanh_appx",
-        "1l_relu",
+        "1l_relu", //15
         "1l_downsample",
         "1l_tanh",
         "2l_relu_sigmoid_small",
         "2l_relu_fc",
-        "2l_relu_small",
+        "2l_relu_small", //20
         "2l_relu_sigmoid",
         "1l_conv",
         "2l_sigmoid_small",
         "2l_relu_sigmoid_conv",
-        "3l_relu_conv_fc",
+        "3l_relu_conv_fc", //25
         "4l_relu_conv_fc",
         "1l_erf",
         "1l_var",
-        "1l_elu", //30
-        "min",
+        "1l_elu",
+        "min", //30
         "max",
         "1l_max_pool",
         "1l_conv_transpose",
-        "1l_upsample", //35
-        "1l_identity",
+        "1l_upsample",
+        "1l_identity", //35
         "idolmodel",
         "trig",
         "prelu_gmm",
-        "lstm", //40
-        "rnn",
+        "lstm",
+        "rnn", //40
         "quantize_dequantize",
         "1l_where",
         "boolean",
         "boolean_identity",
-        "decision_tree", // "variable_cnn",
+        "decision_tree", // 45
         "random_forest",
         "gradient_boosted_trees",
         "1l_topk",
-        "xgboost", //50
-        "lightgbm",
+        "xgboost",
+        "lightgbm", //50
         "hummingbird_decision_tree",
         "oh_decision_tree",
         "linear_svc",
         "gather_elements",
-        "less",
+        "less", //55
         "xgboost_reg",
         "1l_powf",
         "scatter_elements",
-        "1l_linear", //60
-        "linear_regression",
+        "1l_linear",
+        "linear_regression", //60
         "sklearn_mlp",
         "1l_mean",
         "rounding_ops",
         // "mean_as_constrain",
         "arange",
-        "layernorm",
+        "layernorm", //65
         "bitwise_ops",
         "blackman_window",
-        "softsign", //70
+        "softsign", //68
         "softplus",
-        "selu",
+        "selu", //70
         "hard_sigmoid",
         "log_softmax",
         "eye",
         "ltsf",
-        "remainder",
+        "remainder", //75
         "bitshift",
     ];