feat(benchmarks): add basic tensor operation benchmarks

This commit is contained in:
Umut
2021-10-04 14:54:37 +03:00
parent 7e3b0251fc
commit 43743ffdbc
19 changed files with 898 additions and 0 deletions

View File

@@ -0,0 +1,46 @@
# Target: 124 - x (Tensor)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return 124 - x
x = hnp.EncryptedTensor(hnp.UnsignedInteger(6), shape=(3,))
inputset = [
(np.array([36, 50, 24]),),
(np.array([41, 60, 51]),),
(np.array([25, 31, 24]),),
(np.array([34, 47, 27]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 6, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x - [1, 2, 3]
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x - np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]),),
(np.array([1, 3, 5]),),
(np.array([5, 7, 2]),),
(np.array([1, 7, 7]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(3, 2 ** 3, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x - [1, 2, 3] (Broadcasted)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x - np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([[4, 7, 7], [6, 2, 4]]),),
(np.array([[6, 2, 4], [1, 3, 1]]),),
(np.array([[6, 2, 4], [5, 7, 5]]),),
(np.array([[5, 7, 5], [4, 7, 7]]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(3, 2 ** 3, size=(2, 3))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x - 24 (Tensor)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x - 24
x = hnp.EncryptedTensor(hnp.UnsignedInteger(6), shape=(3,))
inputset = [
(np.array([36, 50, 24]),),
(np.array([41, 60, 51]),),
(np.array([25, 31, 24]),),
(np.array([34, 47, 27]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(24, 2 ** 6, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x - y (Broadcasted Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x - y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([6, 2, 4]), np.array([[5, 1, 3], [0, 0, 4]])),
(np.array([1, 3, 1]), np.array([[0, 3, 1], [1, 2, 1]])),
(np.array([5, 1, 2]), np.array([[5, 0, 2], [2, 1, 1]])),
(np.array([0, 7, 7]), np.array([[0, 5, 1], [0, 7, 2]])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(4, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 5, size=(2, 3))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,50 @@
# Target: x - y (Tensor & Scalar)
import random
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x - y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedScalar(hnp.UnsignedInteger(3))
inputset = [
(np.array([6, 2, 4]), 2),
(np.array([1, 3, 1]), 1),
(np.array([5, 4, 7]), 4),
(np.array([5, 7, 6]), 5),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(3, 2 ** 3, size=(3,))
sample_y = random.randint(0, 3)
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x - y (Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x - y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]), np.array([4, 1, 2])),
(np.array([1, 3, 1]), np.array([1, 1, 0])),
(np.array([5, 1, 2]), np.array([4, 1, 1])),
(np.array([0, 7, 7]), np.array([0, 7, 0])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(3, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 4, size=(3,))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x + [1, 2, 3]
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x + np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]),),
(np.array([1, 3, 1]),),
(np.array([5, 1, 2]),),
(np.array([0, 7, 7]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x + [1, 2, 3] (Broadcasted)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x + np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([[0, 7, 7], [6, 2, 4]]),),
(np.array([[6, 2, 4], [1, 3, 1]]),),
(np.array([[6, 2, 4], [5, 1, 2]]),),
(np.array([[5, 1, 2], [0, 7, 7]]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(2, 3))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x + 42 (Tensor)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x + 42
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]),),
(np.array([1, 3, 1]),),
(np.array([5, 1, 2]),),
(np.array([0, 7, 7]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x + y (Broadcasted Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x + y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([6, 2, 4]), np.array([[5, 1, 2], [0, 7, 7]])),
(np.array([1, 3, 1]), np.array([[0, 7, 7], [6, 2, 4]])),
(np.array([5, 1, 2]), np.array([[6, 2, 4], [1, 3, 1]])),
(np.array([0, 7, 7]), np.array([[1, 3, 1], [5, 1, 2]])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 2 ** 3, size=(2, 3))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,50 @@
# Target: x + y (Tensor & Scalar)
import random
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x + y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedScalar(hnp.UnsignedInteger(3))
inputset = [
(np.array([6, 2, 4]), 4),
(np.array([1, 3, 1]), 1),
(np.array([5, 1, 2]), 2),
(np.array([0, 7, 7]), 5),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = random.randint(0, 2 ** 3 - 1)
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x + y (Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x + y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]), np.array([0, 7, 7])),
(np.array([1, 3, 1]), np.array([6, 2, 4])),
(np.array([5, 1, 2]), np.array([1, 3, 1])),
(np.array([0, 7, 7]), np.array([5, 1, 2])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x * [1, 2, 3]
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x * np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]),),
(np.array([1, 3, 1]),),
(np.array([5, 1, 2]),),
(np.array([0, 7, 7]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x * [1, 2, 3] (Broadcasted)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x * np.array([1, 2, 3])
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([[0, 7, 7], [6, 2, 4]]),),
(np.array([[6, 2, 4], [1, 3, 1]]),),
(np.array([[6, 2, 4], [5, 1, 2]]),),
(np.array([[5, 1, 2], [0, 7, 7]]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(2, 3))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,46 @@
# Target: x * 7 (Tensor)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x):
return x * 7
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]),),
(np.array([1, 3, 1]),),
(np.array([5, 1, 2]),),
(np.array([0, 7, 7]),),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x * y (Broadcasted Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x * y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 3))
inputset = [
(np.array([6, 2, 4]), np.array([[5, 1, 2], [0, 7, 7]])),
(np.array([1, 3, 1]), np.array([[0, 7, 7], [6, 2, 4]])),
(np.array([5, 1, 2]), np.array([[6, 2, 4], [1, 3, 1]])),
(np.array([0, 7, 7]), np.array([[1, 3, 1], [5, 1, 2]])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 2 ** 3, size=(2, 3))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,50 @@
# Target: x * y (Tensor & Scalar)
import random
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x * y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedScalar(hnp.UnsignedInteger(3))
inputset = [
(np.array([6, 2, 4]), 4),
(np.array([1, 3, 1]), 1),
(np.array([5, 1, 2]), 2),
(np.array([0, 7, 7]), 5),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = random.randint(0, 5)
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,48 @@
# Target: x * y (Tensors)
import numpy as np
import concrete.numpy as hnp
def main():
def function_to_compile(x, y):
return x * y
x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
y = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(3,))
inputset = [
(np.array([6, 2, 4]), np.array([0, 7, 7])),
(np.array([1, 3, 1]), np.array([6, 2, 4])),
(np.array([5, 1, 2]), np.array([1, 3, 1])),
(np.array([0, 7, 7]), np.array([5, 1, 2])),
]
# Measure: Compilation Time (ms)
engine = hnp.compile_numpy_function(function_to_compile, {"x": x, "y": y}, inputset)
# Measure: End
inputs = []
labels = []
for _ in range(4):
sample_x = np.random.randint(0, 2 ** 3, size=(3,))
sample_y = np.random.randint(0, 2 ** 3, size=(3,))
inputs.append([sample_x, sample_y])
labels.append(function_to_compile(*inputs[-1]))
correct = 0
for input_i, label_i in zip(inputs, labels):
# Measure: Evaluation Time (ms)
result_i = engine.run(*input_i)
# Measure: End
if result_i == label_i:
correct += 1
# Measure: Accuracy (%) = (correct / len(inputs)) * 100
if __name__ == "__main__":
main()