chore: more complete coverage tests (#748)

examples/onnx/1l_lppool/gen.py

@@ -0,0 +1,40 @@
+from torch import nn
+import torch
+import json
+
+class Model(nn.Module):
+    def __init__(self):
+        super(Model, self).__init__()
+        self.layer = nn.LPPool2d(2, 1, (1, 1))
+
+    def forward(self, x):
+        return self.layer(x)[0]
+
+
+circuit = Model()
+
+x = torch.empty(1, 3, 2, 2).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))
+

examples/onnx/1l_lppool/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.7549541592597961, 0.990360677242279, 0.9473411440849304, 0.3951031565666199, 0.8500555753707886, 0.9352139830589294, 0.11867779493331909, 0.9493132829666138, 0.6588345766067505, 0.1933223009109497, 0.12139874696731567, 0.8547163605690002]]}

examples/onnx/celu/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = nn.CELU()(x)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/celu/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.35387128591537476, 0.030473172664642334, 0.08707714080810547, 0.2429301142692566, 0.45228832960128784, 0.496021032333374, 0.13245105743408203, 0.8497090339660645]]}

examples/onnx/clip/gen.py

@@ -0,0 +1,41 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.clamp(x, min=0.4, max=0.8)
+        return m
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 2, 2, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/clip/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.03297048807144165, 0.46362626552581787, 0.6044231057167053, 0.4949902892112732, 0.48823297023773193, 0.6798646450042725, 0.6824942231178284, 0.03491640090942383, 0.19608813524246216, 0.24129939079284668, 0.9769315123558044, 0.6306831240653992, 0.7690497636795044, 0.252221941947937, 0.9167693853378296, 0.3882681131362915, 0.9307044148445129, 0.33559417724609375, 0.7815426588058472, 0.3435332179069519, 0.7871478796005249, 0.12240773439407349, 0.5295405983924866, 0.4874419569969177, 0.08262640237808228, 0.1124718189239502, 0.5834914445877075, 0.30927878618240356, 0.48899340629577637, 0.9376634955406189, 0.21893149614334106, 0.526070773601532]]}

examples/onnx/clip/network.onnx

@@ -0,0 +1,24 @@
+pytorch2.2.1:±
+?/Constant_output_0	/Constant"Constant*
+value*
JÍÌÌ> 

+C/Constant_1_output_0/Constant_1"Constant*
+value*
JÍÌL? 

+F
+input
+/Constant_output_0
+/Constant_1_output_0output/Clip"Clip
+main_graphZ)
+input 
+

+
+batch_size
+
+
+b*
+output 
+

+
+batch_size
+
+
+B

examples/onnx/gru/gen.py

@@ -0,0 +1,41 @@
+import random
+import math
+import numpy as np
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+import json
+
+
+model = nn.GRU(3, 3)  # Input dim is 3, output dim is 3
+x = torch.randn(1, 3)  # make a sequence of length 5
+
+print(x)
+
+# Flips the neural net into inference mode
+model.eval()
+model.to('cpu')
+
+# Export the model
+torch.onnx.export(model,               # model being run
+                  # model input (or a tuple for multiple inputs)
+                  x,
+                  # where to save the model (can be a file or file-like object)
+                  "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=10,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},    # variable length axes
+                                'output': {0: 'batch_size'}})
+
+data_array = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data_json = dict(input_data=[data_array])
+
+print(data_json)
+
+# Serialize data into file:
+json.dump(data_json, open("input.json", 'w'))

examples/onnx/gru/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.4145222008228302, -0.4043896496295929, 0.7545749545097351]]}

examples/onnx/hard_max/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.argmax(x)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/hard_max/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.5505883693695068, 0.0766521692276001, 0.12006187438964844, 0.9497959017753601, 0.9100563526153564, 0.968717098236084, 0.5978299379348755, 0.9419963359832764]]}

examples/onnx/hard_sigmoid/gen.py

@@ -9,7 +9,7 @@ class MyModel(nn.Module):
         super(MyModel, self).__init__()
 
     def forward(self, x):          
-        m = nn.Logsoftmax()(x)
+        m = nn.Hardsigmoid()(x)
         
         return m 
 

examples/onnx/hard_sigmoid/input.json

@@ -1 +1 @@
-{"input_data": [[0.2971532940864563, 0.3465197682380676, 0.05381882190704346, 0.058654189109802246, 0.014198064804077148, 0.06088751554489136, 0.1723427176475525, 0.5115123987197876]]}
+{"input_data": [[0.8326942324638367, 0.2796096205711365, 0.600328266620636, 0.3701696991920471, 0.17832040786743164, 0.6247223019599915, 0.501872718334198, 0.6961578726768494]]}

examples/onnx/hard_sigmoid/network.onnx

@@ -1,4 +1,4 @@
-pytorch2.1.0:<3A>

+pytorch2.2.1:<3A>

 ;
 inputoutput/HardSigmoid"HardSigmoid*
 alpha«ª*> 



examples/onnx/hard_swish/gen.py

@@ -0,0 +1,41 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):
+        m = nn.Hardswish()(x)
+        return m
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/hard_swish/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.6996762752532959, 0.42992985248565674, 0.5102168321609497, 0.5540387630462646, 0.8489438891410828, 0.8533616065979004, 0.36736780405044556, 0.5859147310256958]]}

examples/onnx/hard_swish/network.onnx

@@ -0,0 +1,15 @@
+pytorch2.2.1:{
+&
+inputoutput
+/HardSwish"	HardSwish
+main_graphZ!
+input
+

+
+batch_size
+b"
+output
+

+
+batch_size
+B

examples/onnx/log_softmax/gen.py

@@ -9,7 +9,7 @@ class MyModel(nn.Module):
         super(MyModel, self).__init__()
 
     def forward(self, x):          
-        m = nn.Hardsigmoid()(x)
+        m = nn.LogSoftmax()(x)
         
         return m 
 

examples/onnx/logsumexp/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.logsumexp(x, dim=1)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 2, 2, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/logsumexp/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.7973018884658813, 0.5245689153671265, 0.34149593114852905, 0.1455438733100891, 0.9482707381248474, 0.4221445322036743, 0.001363217830657959, 0.8736765384674072, 0.42954301834106445, 0.7199509739875793, 0.37641745805740356, 0.5920265316963196, 0.42270803451538086, 0.41761744022369385, 0.603948712348938, 0.7250819802284241, 0.047173500061035156, 0.5115441679954529, 0.3743387460708618, 0.16794061660766602, 0.5352339148521423, 0.037976861000061035, 0.65323406457901, 0.5585184097290039, 0.10559147596359253, 0.07827490568161011, 0.6717077493667603, 0.6480781435966492, 0.9780838489532471, 0.8353415131568909, 0.6491701006889343, 0.6573048233985901]]}

examples/onnx/mish/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = nn.Mish()(x)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/mish/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.18563222885131836, 0.4843214750289917, 0.9991059899330139, 0.02534431219100952, 0.8105666041374207, 0.9658406376838684, 0.681107759475708, 0.5365872979164124]]}

examples/onnx/mish/network.onnx

@@ -0,0 +1,19 @@
+pytorch2.2.1:ä

+0
+input/Softplus_output_0	/Softplus"Softplus
+1
+/Softplus_output_0/Tanh_output_0/Tanh"Tanh
+*
+input
+/Tanh_output_0output/Mul"Mul
+main_graphZ!
+input
+

+
+batch_size
+b"
+output
+

+
+batch_size
+B

examples/onnx/reducel1/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.norm(x, p=1, dim=1)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 2, 2, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/reducel1/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.02284395694732666, 0.7941043376922607, 0.07971876859664917, 0.8898420929908752, 0.8233054280281067, 0.11066079139709473, 0.4424799084663391, 0.4355071783065796, 0.6723723411560059, 0.6818525195121765, 0.8726171851158142, 0.17742449045181274, 0.054257750511169434, 0.5775953531265259, 0.7758923172950745, 0.8431423306465149, 0.7602444887161255, 0.29686522483825684, 0.22489851713180542, 0.0675363540649414, 0.981339693069458, 0.15771394968032837, 0.5801441669464111, 0.9044001698493958, 0.49266451597213745, 0.42621421813964844, 0.35345613956451416, 0.042848050594329834, 0.6908614039421082, 0.5422852039337158, 0.01975083351135254, 0.5772860050201416]]}

examples/onnx/reducel2/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.norm(x, p=2, dim=1)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 2, 2, 8).uniform_(0, 1)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/reducel2/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.8709188103675842, 0.11553549766540527, 0.27376580238342285, 0.7518517971038818, 0.7879393100738525, 0.8765475749969482, 0.14315760135650635, 0.8982420563697815, 0.7274006605148315, 0.39007169008255005, 0.729040801525116, 0.11306107044219971, 0.658822774887085, 0.666404664516449, 0.3001367449760437, 0.45343858003616333, 0.7460223436355591, 0.7423691749572754, 0.7544230818748474, 0.5674425959587097, 0.8728761672973633, 0.27062875032424927, 0.1595977544784546, 0.22975260019302368, 0.6711723208427429, 0.8265992403030396, 0.48679041862487793, 0.689740777015686, 0.330846905708313, 0.5630669593811035, 0.8058932423591614, 0.5802426338195801]]}

examples/onnx/tril/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.triu(x)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 3, 3).uniform_(0, 5)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/tril/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.4870188236236572, 2.275230646133423, 3.126268148422241, 0.6412187218666077, 0.9967470169067383, 1.9814395904541016, 1.6355383396148682, 0.6397527456283569, 0.7825168967247009]]}

examples/onnx/triu/gen.py

@@ -0,0 +1,42 @@
+from torch import nn
+import torch
+import json
+import numpy as np
+
+
+class MyModel(nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+
+    def forward(self, x):          
+        m = torch.tril(x)
+        
+        return m 
+
+
+circuit = MyModel()
+
+x = torch.empty(1, 3, 3).uniform_(0, 5)
+
+out = circuit(x)
+
+print(out)
+
+torch.onnx.export(circuit, x, "network.onnx",
+                  export_params=True,        # store the trained parameter weights inside the model file
+                  opset_version=17,          # the ONNX version to export the model to
+                  do_constant_folding=True,  # whether to execute constant folding for optimization
+                  input_names=['input'],   # the model's input names
+                  output_names=['output'],  # the model's output names
+                  dynamic_axes={'input': {0: 'batch_size'},  # variable length axes
+                                'output': {0: 'batch_size'}})
+
+
+d1 = ((x).detach().numpy()).reshape([-1]).tolist()
+
+data = dict(
+    input_data=[d1],
+)
+
+# Serialize data into file:
+json.dump(data, open("input.json", 'w'))

examples/onnx/triu/input.json

@@ -0,0 +1 @@
+{"input_data": [[0.2898547053337097, 1.8070811033248901, 0.30266255140304565, 3.00581955909729, 0.5379888415336609, 1.7057424783706665, 2.415961265563965, 0.589233934879303, 0.03824889659881592]]}

src/circuit/ops/layouts.rs

@@ -2660,13 +2660,43 @@ pub(crate) fn slice<F: PrimeField + TensorType + PartialOrd>(
     end: &usize,
 ) -> Result<ValTensor<F>, Box<dyn Error>> {
     // assigns the instance to the advice.
-    let mut output = region.assign(&config.custom_gates.output, &values[0])?;
-    region.increment(output.len());
+    let mut output = values[0].clone();
+
+    let is_assigned = output.all_prev_assigned();
+    if !is_assigned {
+        output = region.assign(&config.custom_gates.output, &values[0])?;
+        region.increment(output.len());
+    }
+
     output.slice(axis, start, end)?;
 
     Ok(output)
 }
 
+/// Trilu layout
+pub(crate) fn trilu<F: PrimeField + TensorType + PartialOrd>(
+    config: &BaseConfig<F>,
+    region: &mut RegionCtx<F>,
+    values: &[ValTensor<F>; 1],
+    k: &i32,
+    upper: &bool,
+) -> Result<ValTensor<F>, Box<dyn Error>> {
+    // assigns the instance to the advice.
+    let mut output = values[0].clone();
+
+    let is_assigned = output.all_prev_assigned();
+    if !is_assigned {
+        output = region.assign(&config.custom_gates.inputs[0], &values[0])?;
+    }
+
+    let res = tensor::ops::trilu(output.get_inner_tensor()?, *k, *upper)?;
+
+    let output = region.assign(&config.custom_gates.output, &res.into())?;
+    region.increment(output.len());
+
+    Ok(output)
+}
+
 /// Concat layout
 pub(crate) fn concat<F: PrimeField + TensorType + PartialOrd>(
     values: &[ValTensor<F>],

src/circuit/ops/lookup.rs

@@ -123,6 +123,9 @@ pub enum LookupOp {
         scale: utils::F32,
         a: utils::F32,
     },
+    HardSwish {
+        scale: utils::F32,
+    },
 }
 
 impl LookupOp {
@@ -223,6 +226,9 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for LookupOp {
             LookupOp::ATan { scale } => Ok(tensor::ops::nonlinearities::atan(&x, scale.into())),
             LookupOp::ATanh { scale } => Ok(tensor::ops::nonlinearities::atanh(&x, scale.into())),
             LookupOp::Tanh { scale } => Ok(tensor::ops::nonlinearities::tanh(&x, scale.into())),
+            LookupOp::HardSwish { scale } => {
+                Ok(tensor::ops::nonlinearities::hardswish(&x, scale.into()))
+            }
         }?;
 
         let output = res.map(|x| i128_to_felt(x));
@@ -276,6 +282,7 @@ impl<F: PrimeField + TensorType + PartialOrd> Op<F> for LookupOp {
             LookupOp::ASin { scale } => format!("ASIN(scale={})", scale),
             LookupOp::Sinh { scale } => format!("SINH(scale={})", scale),
             LookupOp::ASinh { scale } => format!("ASINH(scale={})", scale),
+            LookupOp::HardSwish { scale } => format!("HARDSWISH(scale={})", scale),
         }
     }
 

src/circuit/ops/poly.rs

@@ -83,6 +83,10 @@ pub enum PolyOp {
     And,
     Or,
     Xor,
+    Trilu {
+        upper: bool,
+        k: i32,
+    },
 }
 
 impl<F: PrimeField + TensorType + PartialOrd + Serialize + for<'de> Deserialize<'de>> Op<F>
@@ -114,7 +118,7 @@ impl<F: PrimeField + TensorType + PartialOrd + Serialize + for<'de> Deserialize<
             PolyOp::Add => "ADD".into(),
             PolyOp::Mult => "MULT".into(),
             PolyOp::Sub => "SUB".into(),
-            PolyOp::Sum { .. } => "SUM".into(),
+            PolyOp::Sum { axes } => format!("SUM (axes={:?})", axes),
             PolyOp::Prod { .. } => "PROD".into(),
             PolyOp::Pow(_) => "POW".into(),
             PolyOp::Conv { .. } => "CONV".into(),
@@ -128,6 +132,7 @@ impl<F: PrimeField + TensorType + PartialOrd + Serialize + for<'de> Deserialize<
             PolyOp::And => "AND".into(),
             PolyOp::Or => "OR".into(),
             PolyOp::Xor => "XOR".into(),
+            PolyOp::Trilu { upper, k } => format!("TRILU (upper={}, k={})", upper, k),
         }
     }
 
@@ -265,6 +270,7 @@ impl<F: PrimeField + TensorType + PartialOrd + Serialize + for<'de> Deserialize<
                 };
                 tensor::ops::scatter_nd(&x, &idx, &src)
             }
+            PolyOp::Trilu { upper, k } => tensor::ops::trilu(&inputs[0], *k, *upper),
         }?;
 
         Ok(ForwardResult { output: res })
@@ -384,6 +390,9 @@ impl<F: PrimeField + TensorType + PartialOrd + Serialize + for<'de> Deserialize<
             PolyOp::Slice { axis, start, end } => {
                 layouts::slice(config, region, values[..].try_into()?, axis, start, end)?
             }
+            PolyOp::Trilu { upper, k } => {
+                layouts::trilu(config, region, values[..].try_into()?, k, upper)?
+            }
         }))
     }
 

src/graph/utilities.rs

@@ -248,6 +248,8 @@ pub fn new_op_from_onnx(
     symbol_values: &SymbolValues,
     rebase_frac_zero_constants: bool,
 ) -> Result<(SupportedOp, Vec<usize>), Box<dyn std::error::Error>> {
+    use tract_onnx::tract_core::ops::array::Trilu;
+
     use crate::circuit::InputType;
 
     let input_scales = inputs
@@ -363,6 +365,26 @@ pub fn new_op_from_onnx(
             SupportedOp::Constant(c)
         }
 
+        "Trilu" => {
+            let op = load_op::<Trilu>(node.op(), idx, node.op().name().to_string())?;
+            let upper = op.upper;
+
+            // assert second input is a constant
+            let diagonal = if let Some(c) = inputs[1].opkind().get_mutable_constant() {
+                inputs[1].decrement_use();
+                deleted_indices.push(1);
+                let raw_values = &c.raw_values;
+                if raw_values.len() != 1 {
+                    return Err(Box::new(GraphError::InvalidDims(idx, "trilu".to_string())));
+                }
+                raw_values[0] as i32
+            } else {
+                return Err("we only support constant inputs for trilu diagonal".into());
+            };
+
+            SupportedOp::Linear(PolyOp::Trilu { upper, k: diagonal })
+        }
+
         "Gather" => {
             if inputs.len() != 2 {
                 return Err(Box::new(GraphError::InvalidDims(idx, "gather".to_string())));
@@ -839,6 +861,9 @@ pub fn new_op_from_onnx(
         }
         "Abs" => SupportedOp::Nonlinear(LookupOp::Abs),
         "Neg" => SupportedOp::Linear(PolyOp::Neg),
+        "HardSwish" => SupportedOp::Nonlinear(LookupOp::HardSwish {
+            scale: scale_to_multiplier(inputs[0].out_scales()[0]).into(),
+        }),
         "Sigmoid" => SupportedOp::Nonlinear(LookupOp::Sigmoid {
             scale: scale_to_multiplier(inputs[0].out_scales()[0]).into(),
         }),

src/tensor/ops.rs

@@ -8,6 +8,142 @@ use maybe_rayon::{
 use std::collections::{HashMap, HashSet};
 pub use std::ops::{Add, Div, Mul, Neg, Sub};
 
+/// Trilu operation.
+/// # Arguments
+/// * `a` - Tensor
+/// * `k` - i32
+/// * `upper` - Boolean
+/// # Examples
+/// ```
+/// use ezkl::tensor::Tensor;
+/// use ezkl::tensor::ops::trilu;
+/// let a = Tensor::<i128>::new(
+///   Some(&[1, 2, 3, 4, 5, 6]),
+/// &[1, 3, 2],
+/// ).unwrap();
+/// let result = trilu(&a, 1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 2, 0, 0, 0, 0]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 4, 5, 6]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 0, 4, 0, 0]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 0, 3, 4, 5, 6]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);  
+///
+/// let result = trilu(&a, -1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 4, 0, 6]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, -1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 0, 3, 0, 5, 6]), &[1, 3, 2]).unwrap();
+/// assert_eq!(result, expected);  
+///
+/// let a = Tensor::<i128>::new(
+///   Some(&[1, 2, 3, 4, 5, 6]),
+/// &[1, 2, 3],
+/// ).unwrap();
+/// let result = trilu(&a, 1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 2, 3, 0, 0, 6]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 0, 4, 5, 6]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 0, 5, 6]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 0, 0, 4, 5, 0]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);  
+///
+/// let result = trilu(&a, -1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 4, 5, 6]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, -1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 0, 0, 4, 0, 0]), &[1, 2, 3]).unwrap();
+/// assert_eq!(result, expected);  
+///
+/// let a = Tensor::<i128>::new(
+///   Some(&[1, 2, 3, 4, 5, 6, 7, 8, 9]),
+/// &[1, 3, 3],
+/// ).unwrap();
+/// let result = trilu(&a, 1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 2, 3, 0, 0, 6, 0, 0, 0]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 0, 4, 5, 6, 7, 8, 9]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 0, 5, 6, 0, 0, 9]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, 0, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 0, 0, 4, 5, 0, 7, 8, 9]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);  
+///
+/// let result = trilu(&a, -1, true).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[1, 2, 3, 4, 5, 6, 0, 8, 9]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);
+///
+/// let result = trilu(&a, -1, false).unwrap();
+/// let expected = Tensor::<i128>::new(Some(&[0, 0, 0, 4, 0, 0, 7, 8, 0]), &[1, 3, 3]).unwrap();
+/// assert_eq!(result, expected);  
+/// ```
+pub fn trilu<T: TensorType + std::marker::Send + std::marker::Sync>(
+    a: &Tensor<T>,
+    k: i32,
+    upper: bool,
+) -> Result<Tensor<T>, TensorError> {
+    let mut output = a.clone();
+
+    // Given a 2-D matrix or batches of 2-D matrices, returns the upper or lower triangular part of the tensor(s).
+    // The attribute “upper” determines whether the upper or lower part is retained.
+    // If set to true, the upper triangular matrix is retained. Lower triangular matrix is retained otherwise.
+    // Default value for the “upper” attribute is true. Trilu takes one input tensor of shape [*, N, M], where * is zero or more batch dimensions.
+    // The upper triangular part consists of the elements on and above the given diagonal (k).
+    // The lower triangular part consists of elements on and below the diagonal. All other elements in the matrix are set to zero.
+
+    let batch_dims = &a.dims()[0..a.dims().len() - 2];
+    let batch_cartiesian = batch_dims.iter().map(|d| 0..*d).multi_cartesian_product();
+
+    for b in batch_cartiesian {
+        for i in 0..a.dims()[1] {
+            for j in 0..a.dims()[2] {
+                let mut coord = b.clone();
+                coord.push(i);
+                coord.push(j);
+                // If k = 0, the triangular part on and above/below the main diagonal is retained.
+
+                if upper {
+                    // If upper is set to true, a positive k retains the upper triangular matrix excluding the main diagonal and (k-1) diagonals above it.
+                    if (j as i32) < (i as i32) + k {
+                        output.set(&coord, T::zero().ok_or(TensorError::Unsupported)?);
+                    }
+                } else {
+                    // If upper is set to false, a positive k retains the lower triangular matrix including the main diagonal and k diagonals above it.
+                    if (j as i32) > (i as i32) + k {
+                        output.set(&coord, T::zero().ok_or(TensorError::Unsupported)?);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(output)
+}
+
 /// IFF operation.
 /// # Arguments
 /// * `mask` - Tensor of 0s and 1s
@@ -3256,6 +3392,47 @@ pub mod nonlinearities {
         .unwrap()
     }
 
+    /// Elementwise applies hardswish to a tensor of integers.
+    /// Hardswish is defined as:
+    // Hardswish(x)={0if x≤−3,xif x≥+3,x⋅(x+3)/6otherwise
+    // Hardswish(x)=⎩
+    // ⎨
+    // ⎧​0xx⋅(x+3)/6​if x≤−3,if x≥+3,otherwise​
+    /// # Arguments
+    ///
+    /// * `a` - Tensor
+    /// * `scale_input` - Single value
+    /// * `scale_output` - Single value
+    /// # Examples
+    /// ```
+    /// use ezkl::tensor::Tensor;
+    /// use ezkl::tensor::ops::nonlinearities::hardswish;
+    /// let x = Tensor::<i128>::new(
+    ///     Some(&[-12, -3, 2, 1, 1, 15]),
+    ///     &[2, 3],
+    /// ).unwrap();
+    /// let result = hardswish(&x, 1.0);
+    /// let expected = Tensor::<i128>::new(Some(&[0, 0, 2, 1, 1, 15]), &[2, 3]).unwrap();
+    ///
+    /// assert_eq!(result, expected);
+    ///
+    /// ```
+    pub fn hardswish(a: &Tensor<i128>, scale_input: f64) -> Tensor<i128> {
+        a.par_enum_map(|_, a_i| {
+            let kix = (a_i as f64) / scale_input;
+            let res = if kix <= -3.0 {
+                0.0
+            } else if kix >= 3.0 {
+                kix
+            } else {
+                kix * (kix + 3.0) / 6.0
+            };
+            let rounded = (res * scale_input).round();
+            Ok::<_, TensorError>(rounded as i128)
+        })
+        .unwrap()
+    }
+
     /// Elementwise applies exponential to a tensor of integers.
     /// # Arguments
     ///

tests/integration_tests.rs

@@ -200,7 +200,7 @@ mod native_tests {
         "1l_tiny_div",
     ];
 
-    const TESTS: [&str; 79] = [
+    const TESTS: [&str; 91] = [
         "1l_mlp", //0
         "1l_slice",
         "1l_concat",
@@ -284,6 +284,18 @@ mod native_tests {
         "bitshift",
         "gather_nd",
         "scatter_nd",
+        "celu",
+        "gru",        // 80
+        "hard_swish", // 81
+        "hard_max",
+        "tril",      // 83
+        "triu",      // 84
+        "logsumexp", // 85
+        "clip",
+        "mish",
+        "reducel1",
+        "reducel2", // 89
+        "1l_lppool",
     ];
 
     const WASM_TESTS: [&str; 46] = [
@@ -522,7 +534,7 @@ mod native_tests {
             }
         });
 
-            seq!(N in 0..=78 {
+            seq!(N in 0..=90 {
 
             #(#[test_case(TESTS[N])])*
             #[ignore]