From c8b95de3e4df7a54ef840b471c13163a335627f4 Mon Sep 17 00:00:00 2001 From: Umut Date: Mon, 11 Oct 2021 18:04:54 +0300 Subject: [PATCH] feat(benchmark): add alerts to benchmarks --- benchmarks/124_minus_x.py | 1 + benchmarks/124_minus_x_tensor.py | 1 + benchmarks/linear_regression.py | 1 + benchmarks/logistic_regression.py | 1 + benchmarks/single_table_lookup.py | 1 + benchmarks/x_minus_1_2_3.py | 1 + benchmarks/x_minus_1_2_3_broadcasted.py | 1 + benchmarks/x_minus_24.py | 1 + benchmarks/x_minus_24_tensor.py | 1 + benchmarks/x_minus_y.py | 1 + benchmarks/x_minus_y_broadcasted_tensors.py | 1 + benchmarks/x_minus_y_tensor_and_scalar.py | 1 + benchmarks/x_minus_y_tensors.py | 1 + benchmarks/x_plus_1_2_3.py | 1 + benchmarks/x_plus_1_2_3_broadcasted.py | 1 + benchmarks/x_plus_42.py | 1 + benchmarks/x_plus_42_tensor.py | 1 + benchmarks/x_plus_y.py | 1 + benchmarks/x_plus_y_broadcasted_tensors.py | 1 + benchmarks/x_plus_y_tensor_and_scalar.py | 1 + benchmarks/x_plus_y_tensors.py | 1 + benchmarks/x_times_1_2_3.py | 1 + benchmarks/x_times_1_2_3_broadcasted.py | 1 + benchmarks/x_times_7.py | 1 + benchmarks/x_times_7_tensor.py | 1 + benchmarks/x_times_y.py | 1 + benchmarks/x_times_y_broadcasted_tensors.py | 1 + benchmarks/x_times_y_tensor_and_scalar.py | 1 + benchmarks/x_times_y_tensors.py | 1 + benchmarks/x_to_the_power_of_2.py | 1 + 30 files changed, 30 insertions(+) diff --git a/benchmarks/124_minus_x.py b/benchmarks/124_minus_x.py index 1b8e4f13f..fb192944e 100644 --- a/benchmarks/124_minus_x.py +++ b/benchmarks/124_minus_x.py @@ -40,6 +40,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/124_minus_x_tensor.py b/benchmarks/124_minus_x_tensor.py index ed91bc310..eab8fcafe 100644 --- a/benchmarks/124_minus_x_tensor.py +++ b/benchmarks/124_minus_x_tensor.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/linear_regression.py b/benchmarks/linear_regression.py index c74b2d0b6..e71f63cb7 100644 --- a/benchmarks/linear_regression.py +++ b/benchmarks/linear_regression.py @@ -191,6 +191,7 @@ def main(): # Measure: Non Homomorphic Loss = non_homomorphic_loss # Measure: Homomorphic Loss = homomorphic_loss # Measure: Relative Loss Difference Between Homomorphic and Non Homomorphic Implementation (%) = difference + # Alert: Relative Loss Difference Between Homomorphic and Non Homomorphic Implementation (%) > 20 if __name__ == "__main__": diff --git a/benchmarks/logistic_regression.py b/benchmarks/logistic_regression.py index 664ac700b..1d7de17c3 100644 --- a/benchmarks/logistic_regression.py +++ b/benchmarks/logistic_regression.py @@ -264,6 +264,7 @@ def main(): print(f"Accuracy: {accuracy:.2f}%") # Measure: Accuracy (%) = accuracy + # Alert: Accuracy (%) < 85 if __name__ == "__main__": diff --git a/benchmarks/single_table_lookup.py b/benchmarks/single_table_lookup.py index 98f3018a6..0f0541398 100644 --- a/benchmarks/single_table_lookup.py +++ b/benchmarks/single_table_lookup.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_1_2_3.py b/benchmarks/x_minus_1_2_3.py index 746f69d39..0e696dc8a 100644 --- a/benchmarks/x_minus_1_2_3.py +++ b/benchmarks/x_minus_1_2_3.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_1_2_3_broadcasted.py b/benchmarks/x_minus_1_2_3_broadcasted.py index b78e4a2f4..f1473f062 100644 --- a/benchmarks/x_minus_1_2_3_broadcasted.py +++ b/benchmarks/x_minus_1_2_3_broadcasted.py @@ -43,6 +43,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_24.py b/benchmarks/x_minus_24.py index 9a39d0b22..3bca798d1 100644 --- a/benchmarks/x_minus_24.py +++ b/benchmarks/x_minus_24.py @@ -40,6 +40,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_24_tensor.py b/benchmarks/x_minus_24_tensor.py index b88be1f8d..3961c94bd 100644 --- a/benchmarks/x_minus_24_tensor.py +++ b/benchmarks/x_minus_24_tensor.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_y.py b/benchmarks/x_minus_y.py index 47c24a09e..b7d5db488 100644 --- a/benchmarks/x_minus_y.py +++ b/benchmarks/x_minus_y.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_y_broadcasted_tensors.py b/benchmarks/x_minus_y_broadcasted_tensors.py index d586a8ab2..808575c49 100644 --- a/benchmarks/x_minus_y_broadcasted_tensors.py +++ b/benchmarks/x_minus_y_broadcasted_tensors.py @@ -46,6 +46,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_y_tensor_and_scalar.py b/benchmarks/x_minus_y_tensor_and_scalar.py index 27e825ca4..8e69743b9 100644 --- a/benchmarks/x_minus_y_tensor_and_scalar.py +++ b/benchmarks/x_minus_y_tensor_and_scalar.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_minus_y_tensors.py b/benchmarks/x_minus_y_tensors.py index 5932e97c2..3858563f3 100644 --- a/benchmarks/x_minus_y_tensors.py +++ b/benchmarks/x_minus_y_tensors.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_1_2_3.py b/benchmarks/x_plus_1_2_3.py index e9339d97f..7503d78cd 100644 --- a/benchmarks/x_plus_1_2_3.py +++ b/benchmarks/x_plus_1_2_3.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_1_2_3_broadcasted.py b/benchmarks/x_plus_1_2_3_broadcasted.py index 0095cb8c1..36132d3d6 100644 --- a/benchmarks/x_plus_1_2_3_broadcasted.py +++ b/benchmarks/x_plus_1_2_3_broadcasted.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_42.py b/benchmarks/x_plus_42.py index 95f7be9ed..d74ff023e 100644 --- a/benchmarks/x_plus_42.py +++ b/benchmarks/x_plus_42.py @@ -40,6 +40,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_42_tensor.py b/benchmarks/x_plus_42_tensor.py index 6ca272380..a709b788e 100644 --- a/benchmarks/x_plus_42_tensor.py +++ b/benchmarks/x_plus_42_tensor.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_y.py b/benchmarks/x_plus_y.py index a4c809b9c..39aae0377 100644 --- a/benchmarks/x_plus_y.py +++ b/benchmarks/x_plus_y.py @@ -42,6 +42,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_y_broadcasted_tensors.py b/benchmarks/x_plus_y_broadcasted_tensors.py index 91756ec8b..b2408d13f 100644 --- a/benchmarks/x_plus_y_broadcasted_tensors.py +++ b/benchmarks/x_plus_y_broadcasted_tensors.py @@ -46,6 +46,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_y_tensor_and_scalar.py b/benchmarks/x_plus_y_tensor_and_scalar.py index c44dd9e81..94224b449 100644 --- a/benchmarks/x_plus_y_tensor_and_scalar.py +++ b/benchmarks/x_plus_y_tensor_and_scalar.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_plus_y_tensors.py b/benchmarks/x_plus_y_tensors.py index 4508d2ecc..836d89c97 100644 --- a/benchmarks/x_plus_y_tensors.py +++ b/benchmarks/x_plus_y_tensors.py @@ -46,6 +46,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_1_2_3.py b/benchmarks/x_times_1_2_3.py index 1f3b4bc85..040297c88 100644 --- a/benchmarks/x_times_1_2_3.py +++ b/benchmarks/x_times_1_2_3.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_1_2_3_broadcasted.py b/benchmarks/x_times_1_2_3_broadcasted.py index cefe9f8ec..77d56aa57 100644 --- a/benchmarks/x_times_1_2_3_broadcasted.py +++ b/benchmarks/x_times_1_2_3_broadcasted.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_7.py b/benchmarks/x_times_7.py index f5338bfb5..156aafa1d 100644 --- a/benchmarks/x_times_7.py +++ b/benchmarks/x_times_7.py @@ -40,6 +40,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_7_tensor.py b/benchmarks/x_times_7_tensor.py index 7798ad568..c92d02031 100644 --- a/benchmarks/x_times_7_tensor.py +++ b/benchmarks/x_times_7_tensor.py @@ -41,6 +41,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_y.py b/benchmarks/x_times_y.py index 562289db3..50dc97d13 100644 --- a/benchmarks/x_times_y.py +++ b/benchmarks/x_times_y.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_y_broadcasted_tensors.py b/benchmarks/x_times_y_broadcasted_tensors.py index a804ad773..c830ea77c 100644 --- a/benchmarks/x_times_y_broadcasted_tensors.py +++ b/benchmarks/x_times_y_broadcasted_tensors.py @@ -46,6 +46,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_y_tensor_and_scalar.py b/benchmarks/x_times_y_tensor_and_scalar.py index 906c31fa1..7b7bad7e4 100644 --- a/benchmarks/x_times_y_tensor_and_scalar.py +++ b/benchmarks/x_times_y_tensor_and_scalar.py @@ -45,6 +45,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_times_y_tensors.py b/benchmarks/x_times_y_tensors.py index e4e5976c0..7d561eb4a 100644 --- a/benchmarks/x_times_y_tensors.py +++ b/benchmarks/x_times_y_tensors.py @@ -46,6 +46,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__": diff --git a/benchmarks/x_to_the_power_of_2.py b/benchmarks/x_to_the_power_of_2.py index da3505f64..4391f6ad0 100644 --- a/benchmarks/x_to_the_power_of_2.py +++ b/benchmarks/x_to_the_power_of_2.py @@ -40,6 +40,7 @@ def main(): correct += 1 # Measure: Accuracy (%) = (correct / len(inputs)) * 100 + # Alert: Accuracy (%) != 100 if __name__ == "__main__":