feat(benchmarks): add support for full and unit targets

benchmarks/124_minus_x.py

@@ -1,4 +1,4 @@
-# Target: 124 - x
+# Unit Target: 124 - x
 
 import random
 

benchmarks/124_minus_x_tensor.py

@@ -1,4 +1,4 @@
-# Target: 124 - x (Tensor)
+# Unit Target: 124 - x (Tensor)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/linear_regression.py

@@ -1,4 +1,4 @@
-# Target: Linear Regression
+# Full Target: Linear Regression
 
 # Disable line length warnings as we have a looooong metric...
 # flake8: noqa: E501

benchmarks/logistic_regression.py

@@ -1,4 +1,4 @@
-# Target: Logistic Regression
+# Full Target: Logistic Regression
 
 import numpy as np
 import torch

benchmarks/single_table_lookup.py

@@ -1,4 +1,4 @@
-# Target: Single Table Lookup
+# Unit Target: Single Table Lookup
 
 import random
 

benchmarks/x_minus_1_2_3.py

@@ -1,4 +1,4 @@
-# Target: x - [1, 2, 3]
+# Unit Target: x - [1, 2, 3]
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_minus_1_2_3_broadcasted.py

@@ -1,4 +1,4 @@
-# Target: x - [1, 2, 3] (Broadcasted)
+# Unit Target: x - [1, 2, 3] (Broadcasted)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_minus_24.py

@@ -1,4 +1,4 @@
-# Target: x - 24
+# Unit Target: x - 24
 
 import random
 

benchmarks/x_minus_24_tensor.py

@@ -1,4 +1,4 @@
-# Target: x - 24 (Tensor)
+# Unit Target: x - 24 (Tensor)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_minus_y.py

@@ -1,4 +1,4 @@
-# Target: x - y
+# Unit Target: x - y
 
 import itertools
 import random

benchmarks/x_minus_y_broadcasted_tensors.py

@@ -1,4 +1,4 @@
-# Target: x - y (Broadcasted Tensors)
+# Unit Target: x - y (Broadcasted Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_minus_y_tensor_and_scalar.py

@@ -1,4 +1,4 @@
-# Target: x - y (Tensor & Scalar)
+# Unit Target: x - y (Tensor & Scalar)
 
 import random
 

benchmarks/x_minus_y_tensors.py

@@ -1,4 +1,4 @@
-# Target: x - y (Tensors)
+# Unit Target: x - y (Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_plus_1_2_3.py

@@ -1,4 +1,4 @@
-# Target: x + [1, 2, 3]
+# Unit Target: x + [1, 2, 3]
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_plus_1_2_3_broadcasted.py

@@ -1,4 +1,4 @@
-# Target: x + [1, 2, 3] (Broadcasted)
+# Unit Target: x + [1, 2, 3] (Broadcasted)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_plus_42.py

@@ -1,4 +1,4 @@
-# Target: x + 42
+# Unit Target: x + 42
 
 import random
 

benchmarks/x_plus_42_tensor.py

@@ -1,4 +1,4 @@
-# Target: x + 42 (Tensor)
+# Unit Target: x + 42 (Tensor)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_plus_y.py

@@ -1,4 +1,4 @@
-# Target: x + y
+# Unit Target: x + y
 
 import random
 

benchmarks/x_plus_y_broadcasted_tensors.py

@@ -1,4 +1,4 @@
-# Target: x + y (Broadcasted Tensors)
+# Unit Target: x + y (Broadcasted Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_plus_y_tensor_and_scalar.py

@@ -1,4 +1,4 @@
-# Target: x + y (Tensor & Scalar)
+# Unit Target: x + y (Tensor & Scalar)
 
 import random
 

benchmarks/x_plus_y_tensors.py

@@ -1,4 +1,4 @@
-# Target: x + y (Tensors)
+# Unit Target: x + y (Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_times_1_2_3.py

@@ -1,4 +1,4 @@
-# Target: x * [1, 2, 3]
+# Unit Target: x * [1, 2, 3]
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_times_1_2_3_broadcasted.py

@@ -1,4 +1,4 @@
-# Target: x * [1, 2, 3] (Broadcasted)
+# Unit Target: x * [1, 2, 3] (Broadcasted)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_times_7.py

@@ -1,4 +1,4 @@
-# Target: x * 7
+# Unit Target: x * 7
 
 import random
 

benchmarks/x_times_7_tensor.py

@@ -1,4 +1,4 @@
-# Target: x * 7 (Tensor)
+# Unit Target: x * 7 (Tensor)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_times_y.py

@@ -1,4 +1,4 @@
-# Target: x * y
+# Unit Target: x * y
 
 import itertools
 import random

benchmarks/x_times_y_broadcasted_tensors.py

@@ -1,4 +1,4 @@
-# Target: x * y (Broadcasted Tensors)
+# Unit Target: x * y (Broadcasted Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_times_y_tensor_and_scalar.py

@@ -1,4 +1,4 @@
-# Target: x * y (Tensor & Scalar)
+# Unit Target: x * y (Tensor & Scalar)
 
 import random
 

benchmarks/x_times_y_tensors.py

@@ -1,4 +1,4 @@
-# Target: x * y (Tensors)
+# Unit Target: x * y (Tensors)
 
 import numpy as np
 from common import BENCHMARK_CONFIGURATION

benchmarks/x_to_the_power_of_2.py

@@ -1,4 +1,4 @@
-# Target: x**2
+# Unit Target: x**2
 
 import random
 

script/progress_tracker_utils/measure.py

@@ -284,15 +284,8 @@ def perform_measurements(path, script, target_id, metrics, samples, result):
         del result["targets"][target_id]["measurements"]
 
 
-def main():
+def main(args):
     """Measurement script for the progress tracker"""
-    parser = argparse.ArgumentParser(description="Measurement script for the progress tracker")
-
-    parser.add_argument("base", type=str, help="directory which contains the benchmarks")
-    parser.add_argument("--samples", type=int, default=30, help="number of samples to take")
-    parser.add_argument("--keep", action="store_true", help="flag to keep measurement scripts")
-
-    args = parser.parse_args()
 
     base = pathlib.Path(args.base)
     samples = args.samples
@@ -328,7 +321,15 @@ def main():
                 break
 
         # Check whether the script is a target or not
-        if not first_line.startswith("# Target:"):
+        if first_line.startswith("# Unit Target:"):
+            # Extract target name
+            target_name = first_line.replace("# Unit Target:", "").strip()
+            is_unit = True
+        elif first_line.startswith("# Full Target:"):
+            # Extract target name
+            target_name = first_line.replace("# Full Target:", "").strip()
+            is_unit = False
+        else:
             print()
             print(path)
             print("-" * len(str(path)))
@@ -342,8 +343,7 @@ def main():
             print()
             continue
 
-        # Extract target name and id
-        target_name = first_line.replace("# Target:", "").strip()
+        # Extract target id
         target_id = name_to_id(target_name)
 
         # Check whether the target is already registered
@@ -371,6 +371,7 @@ def main():
             "measurements": {},
             "alerts": alerts,
             "code": "\n".join(lines),
+            "isUnit": is_unit,
         }
 
         # Perform and save measurements
@@ -388,4 +389,10 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser(description="Measurement script for the progress tracker")
+
+    parser.add_argument("base", type=str, help="directory which contains the benchmarks")
+    parser.add_argument("--samples", type=int, default=30, help="number of samples to take")
+    parser.add_argument("--keep", action="store_true", help="flag to keep measurement scripts")
+
+    main(parser.parse_args())