From 39ad5bd894bdf4d75f275c9ead5ba5c3c64583c9 Mon Sep 17 00:00:00 2001 From: Umut Date: Thu, 4 Nov 2021 16:50:06 +0300 Subject: [PATCH] feat(benchmarks): add dynamic indexing benchmarks --- benchmarks/x_index_colon_and_y_clear.py | 55 +++++++++++++++++ benchmarks/x_index_colon_and_y_encrypted.py | 55 +++++++++++++++++ ...x_colon_colon_minus_1_and_y_and_z_clear.py | 61 +++++++++++++++++++ ...lon_colon_minus_1_and_y_and_z_encrypted.py | 61 +++++++++++++++++++ benchmarks/x_index_y_and_1_colon_clear.py | 55 +++++++++++++++++ benchmarks/x_index_y_and_1_colon_encrypted.py | 55 +++++++++++++++++ benchmarks/x_index_y_and_colon_and_z_clear.py | 61 +++++++++++++++++++ .../x_index_y_and_colon_and_z_encrypted.py | 61 +++++++++++++++++++ benchmarks/x_index_y_and_colon_clear.py | 55 +++++++++++++++++ benchmarks/x_index_y_and_colon_encrypted.py | 55 +++++++++++++++++ benchmarks/x_index_y_and_z_and_0_clear.py | 61 +++++++++++++++++++ benchmarks/x_index_y_and_z_and_0_encrypted.py | 61 +++++++++++++++++++ benchmarks/x_index_y_and_z_clear.py | 61 +++++++++++++++++++ benchmarks/x_index_y_and_z_encrypted.py | 61 +++++++++++++++++++ benchmarks/x_index_y_clear.py | 55 +++++++++++++++++ benchmarks/x_index_y_encrypted.py | 55 +++++++++++++++++ 16 files changed, 928 insertions(+) create mode 100644 benchmarks/x_index_colon_and_y_clear.py create mode 100644 benchmarks/x_index_colon_and_y_encrypted.py create mode 100644 benchmarks/x_index_colon_colon_minus_1_and_y_and_z_clear.py create mode 100644 benchmarks/x_index_colon_colon_minus_1_and_y_and_z_encrypted.py create mode 100644 benchmarks/x_index_y_and_1_colon_clear.py create mode 100644 benchmarks/x_index_y_and_1_colon_encrypted.py create mode 100644 benchmarks/x_index_y_and_colon_and_z_clear.py create mode 100644 benchmarks/x_index_y_and_colon_and_z_encrypted.py create mode 100644 benchmarks/x_index_y_and_colon_clear.py create mode 100644 benchmarks/x_index_y_and_colon_encrypted.py create mode 100644 benchmarks/x_index_y_and_z_and_0_clear.py create mode 100644 benchmarks/x_index_y_and_z_and_0_encrypted.py create mode 100644 benchmarks/x_index_y_and_z_clear.py create mode 100644 benchmarks/x_index_y_and_z_encrypted.py create mode 100644 benchmarks/x_index_y_clear.py create mode 100644 benchmarks/x_index_y_encrypted.py diff --git a/benchmarks/x_index_colon_and_y_clear.py b/benchmarks/x_index_colon_and_y_clear.py new file mode 100644 index 000000000..0e6712f01 --- /dev/null +++ b/benchmarks/x_index_colon_and_y_clear.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[:, y] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[:, y] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 4)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(2, 4)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(2, 4)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_colon_and_y_encrypted.py b/benchmarks/x_index_colon_and_y_encrypted.py new file mode 100644 index 000000000..3b65e05b9 --- /dev/null +++ b/benchmarks/x_index_colon_and_y_encrypted.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[:, y] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[:, y] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(2, 4)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(2, 4)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(2, 4)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_clear.py b/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_clear.py new file mode 100644 index 000000000..7fcfb68b4 --- /dev/null +++ b/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_clear.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[::-1, y, z] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[::-1, y, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(5, 4, 2)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + z = hnp.ClearScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(5, 4, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(5, 4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_encrypted.py b/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_encrypted.py new file mode 100644 index 000000000..632051c50 --- /dev/null +++ b/benchmarks/x_index_colon_colon_minus_1_and_y_and_z_encrypted.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[::-1, y, z] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[::-1, y, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(5, 4, 2)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + z = hnp.EncryptedScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(5, 4, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(5, 4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_1_colon_clear.py b/benchmarks/x_index_y_and_1_colon_clear.py new file mode 100644 index 000000000..359f26054 --- /dev/null +++ b/benchmarks/x_index_y_and_1_colon_clear.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y, 1:] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y, 1:] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 5)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4, 5)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 5)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_1_colon_encrypted.py b/benchmarks/x_index_y_and_1_colon_encrypted.py new file mode 100644 index 000000000..3a3ea62ae --- /dev/null +++ b/benchmarks/x_index_y_and_1_colon_encrypted.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y, 1:] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y, 1:] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 5)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4, 5)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 5)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_colon_and_z_clear.py b/benchmarks/x_index_y_and_colon_and_z_clear.py new file mode 100644 index 000000000..69542b101 --- /dev/null +++ b/benchmarks/x_index_y_and_colon_and_z_clear.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, :, z] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, :, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 5, 2)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + z = hnp.ClearScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 5, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 5, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_colon_and_z_encrypted.py b/benchmarks/x_index_y_and_colon_and_z_encrypted.py new file mode 100644 index 000000000..ab3b7e511 --- /dev/null +++ b/benchmarks/x_index_y_and_colon_and_z_encrypted.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, :, z] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, :, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 5, 2)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + z = hnp.EncryptedScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 5, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 5, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_colon_clear.py b/benchmarks/x_index_y_and_colon_clear.py new file mode 100644 index 000000000..8c6870604 --- /dev/null +++ b/benchmarks/x_index_y_and_colon_clear.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y, :] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y, :] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4, 2)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_colon_encrypted.py b/benchmarks/x_index_y_and_colon_encrypted.py new file mode 100644 index 000000000..6c230664d --- /dev/null +++ b/benchmarks/x_index_y_and_colon_encrypted.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y, :] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y, :] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4, 2)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_z_and_0_clear.py b/benchmarks/x_index_y_and_z_and_0_clear.py new file mode 100644 index 000000000..ea3921bb5 --- /dev/null +++ b/benchmarks/x_index_y_and_z_and_0_clear.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, z, 0] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, z, 0] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2, 5)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + z = hnp.ClearScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 2, 5)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2, 5)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_z_and_0_encrypted.py b/benchmarks/x_index_y_and_z_and_0_encrypted.py new file mode 100644 index 000000000..d9a5fecba --- /dev/null +++ b/benchmarks/x_index_y_and_z_and_0_encrypted.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, z, 0] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, z, 0] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2, 5)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + z = hnp.EncryptedScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 2, 5)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2, 5)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_z_clear.py b/benchmarks/x_index_y_and_z_clear.py new file mode 100644 index 000000000..a3cee932b --- /dev/null +++ b/benchmarks/x_index_y_and_z_clear.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, z] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + z = hnp.ClearScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_and_z_encrypted.py b/benchmarks/x_index_y_and_z_encrypted.py new file mode 100644 index 000000000..4f525d22c --- /dev/null +++ b/benchmarks/x_index_y_and_z_encrypted.py @@ -0,0 +1,61 @@ +# bench: Unit Target: x[y, z] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y, z): + return x[y, z] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4, 2)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + z = hnp.EncryptedScalar(hnp.UnsignedInteger(1)) + + inputset = [ + ( + np.random.randint(0, 2 ** 3, size=(4, 2)), + random.randint(0, (2 ** 2) - 1), + random.randint(0, (2 ** 1) - 1), + ) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y, "z": z}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4, 2)) + sample_y = random.randint(0, (2 ** 2) - 1) + sample_z = random.randint(0, (2 ** 1) - 1) + + inputs.append([sample_x, sample_y, sample_z]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_clear.py b/benchmarks/x_index_y_clear.py new file mode 100644 index 000000000..5e4f4ee6c --- /dev/null +++ b/benchmarks/x_index_y_clear.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y] (Clear) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4,)) + y = hnp.ClearScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4,)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4,)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main() diff --git a/benchmarks/x_index_y_encrypted.py b/benchmarks/x_index_y_encrypted.py new file mode 100644 index 000000000..76db6bbc3 --- /dev/null +++ b/benchmarks/x_index_y_encrypted.py @@ -0,0 +1,55 @@ +# bench: Unit Target: x[y] (Encrypted) + +import random + +import numpy as np +from common import BENCHMARK_CONFIGURATION + +import concrete.numpy as hnp + + +def main(): + def function_to_compile(x, y): + return x[y] + + x = hnp.EncryptedTensor(hnp.UnsignedInteger(3), shape=(4,)) + y = hnp.EncryptedScalar(hnp.UnsignedInteger(2)) + + inputset = [ + (np.random.randint(0, 2 ** 3, size=(4,)), random.randint(0, (2 ** 2) - 1)) + for _ in range(32) + ] + + # bench: Measure: Compilation Time (ms) + engine = hnp.compile_numpy_function( + function_to_compile, + {"x": x, "y": y}, + inputset, + compilation_configuration=BENCHMARK_CONFIGURATION, + ) + # bench: Measure: End + + inputs = [] + labels = [] + for _ in range(100): + sample_x = np.random.randint(0, 2 ** 3, size=(4,)) + sample_y = random.randint(0, (2 ** 2) - 1) + + inputs.append([sample_x, sample_y]) + labels.append(function_to_compile(*inputs[-1])) + + correct = 0 + for input_i, label_i in zip(inputs, labels): + # bench: Measure: Evaluation Time (ms) + result_i = engine.run(*input_i) + # bench: Measure: End + + if result_i == label_i: + correct += 1 + + # bench: Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # bench: Alert: Accuracy (%) != 100 + + +if __name__ == "__main__": + main()