Add metadata to benchmark results. (#297)

shark/shark_benchmark_runner.py

@@ -71,7 +71,7 @@ class SharkBenchmarkRunner(SharkRunner):
             input_tensors,
             mlir_dialect=self.mlir_dialect,
         )
-        # print(self.benchmark_cl)
+        print(self.benchmark_cl)
 
     def benchmark_frontend(self, modelname):
         if self.mlir_dialect in ["linalg", "torch"]:
@@ -236,6 +236,34 @@ for currently supported models. Exiting benchmark ONNX."
                 result[0]["average_latency_ms"],
             ]
 
+    def get_metadata(self, modelname):
+        with open("./tank/pytorch/torch_model_list.csv", mode="r") as csvfile:
+            torch_reader = csv.reader(csvfile, delimiter=",")
+            fields = next(torch_reader)
+            for row in torch_reader:
+                torch_model_name = row[0]
+                param_count = row[4]
+                model_tags = row[5]
+                model_notes = row[6]
+                if torch_model_name == modelname:
+                    return [param_count, model_tags, model_notes]
+
+    def compare_bench_results(self, baseline: str, result: str):
+        # Takes two numbers represented as strings and returns "<n>x slower/faster", as in "result is <n>x slower than baseline".
+        a = float(baseline)
+        b = float(result)
+        if a < b:
+            # result slower than baseline
+            comparison = (b - a) / a
+            comp_str = f"{round(comparison, 2)}x slower"
+        elif a > b:
+            # result faster than baseline
+            comparison = a / b
+            comp_str = f"{round(comparison, 2)}x faster"
+        else:
+            comp_str = "equal"
+        return comp_str
+
     def benchmark_all_csv(
         self, inputs: tuple, modelname, dynamic, device_str, frontend
     ):
@@ -243,12 +271,17 @@ for currently supported models. Exiting benchmark ONNX."
         field_names = [
             "model",
             "engine",
-            "dynamic",
             "dialect",
             "device",
+            "shape_type",
+            "data_type",
             "iter/sec",
             "ms/iter",
+            "vs. PyTorch/TF",
             "iterations",
+            "param_count",
+            "tags",
+            "notes",
             "datetime",
         ]
         engines = ["frontend", "shark_python", "shark_iree_c"]
@@ -265,10 +298,11 @@ for currently supported models. Exiting benchmark ONNX."
             bench_result = {}
             bench_result["model"] = modelname
             if dynamic == True:
-                bench_result["dynamic"] = "True"
+                bench_result["shape_type"] = "dynamic"
             else:
-                bench_result["dynamic"] = "False"
+                bench_result["shape_type"] = "static"
             bench_result["device"] = device_str
+            bench_result["data_type"] = inputs[0].dtype
             for e in engines:
                 if e == "frontend":
                     bench_result["engine"] = frontend
@@ -276,18 +310,50 @@ for currently supported models. Exiting benchmark ONNX."
                         bench_result["iter/sec"],
                         bench_result["ms/iter"],
                     ) = self.benchmark_frontend(modelname)
+                    self.frontend_result = bench_result["ms/iter"]
+                    bench_result["vs. PyTorch/TF"] = "="
+                    (
+                        bench_result["param_count"],
+                        bench_result["tags"],
+                        bench_result["notes"],
+                    ) = self.get_metadata(modelname)
+
                 elif e == "shark_python":
                     bench_result["engine"] = "shark_python"
                     (
                         bench_result["iter/sec"],
                         bench_result["ms/iter"],
                     ) = self.benchmark_python(inputs)
+
+                    bench_result[
+                        "vs. PyTorch/TF"
+                    ] = self.compare_bench_results(
+                        self.frontend_result, bench_result["ms/iter"]
+                    )
+                    (
+                        bench_result["param_count"],
+                        bench_result["tags"],
+                        bench_result["notes"],
+                    ) = ["", "", ""]
+
                 elif e == "shark_iree_c":
                     bench_result["engine"] = "shark_iree_c"
                     (
                         bench_result["iter/sec"],
                         bench_result["ms/iter"],
                     ) = self.benchmark_c()
+
+                    bench_result[
+                        "vs. PyTorch/TF"
+                    ] = self.compare_bench_results(
+                        self.frontend_result, bench_result["ms/iter"]
+                    )
+                    (
+                        bench_result["param_count"],
+                        bench_result["tags"],
+                        bench_result["notes"],
+                    ) = ["", "", ""]
+
                 elif e == "onnxruntime":
                     bench_result["engine"] = "onnxruntime"
                     (

tank/pytorch/bench_results.csv

@@ -1 +0,0 @@
-platform,model,dynamic,device,iter/sec,ms/iter,datetime

tank/pytorch/torch_model_list.csv

@@ -1,18 +1,18 @@
-model_name, use_tracing, model_type, dynamic
-microsoft/MiniLM-L12-H384-uncased,True,hf,True
-albert-base-v2,True,hf,True
-bert-base-uncased,True,hf,True
-bert-base-cased,True,hf,True
-google/mobilebert-uncased,True,hf,True
-alexnet,False,vision,True
-resnet18,False,vision,True
-resnet50,False,vision,True
-resnet101,False,vision,True
-squeezenet1_0,False,vision,True
-wide_resnet50_2,False,vision,True
-mobilenet_v3_small,False,vision,True
-google/vit-base-patch16-224,True,hf_img_cls,False
-microsoft/resnet-50,True,hf_img_cls,False
-facebook/deit-small-distilled-patch16-224,True,hf_img_cls,False
-microsoft/beit-base-patch16-224-pt22k-ft22k,True,hf_img_cls,False
-nvidia/mit-b0,True,hf_img_cls,False
+model_name, use_tracing, model_type, dynamic, param_count, tags, notes
+microsoft/MiniLM-L12-H384-uncased,True,hf,True,66M,"nlp,bert-variant,transformer-encoder","Large version has 12 layers, 384 hidden size,Smaller than BERTbase (66M params vs 109M params)"
+albert-base-v2,True,hf,True,11M,"nlp,bert-variant,transformer-encoder","12 layers, 128 embedding dim, 768 hidden dim, 12 attention heads,Smaller than BERTbase (11M params vs 109M params),Uses weight sharing to reduce # params but computational cost is similar to BERT."
+bert-base-uncased,True,hf,True,109M,"nlp,bert-variant,transformer-encoder","12 layers, 768 hidden, 12 attention heads"
+bert-base-cased,True,hf,True,109M,"nlp,bert-variant,transformer-encoder","12 layers, 768 hidden, 12 attention heads"
+google/mobilebert-uncased,True,hf,True,25M,"nlp,bert-variant,transformer-encoder,mobile","24 layers, 512 hidden size, 128 embedding"
+alexnet,False,vision,True,61M,"cnn,parallel-layers","The CNN that revolutionized computer vision (move away from hand-crafted features to neural networks),10 years old now and probably no longer used in prod."
+resnet18,False,vision,True,11M,"cnn,image-classification,residuals,resnet-variant","1 7x7 conv2d and the rest are 3x3 conv2d"
+resnet50,False,vision,True,23M,"cnn,image-classification,residuals,resnet-variant","Bottlenecks with only conv2d (1x1 conv -> 3x3 conv -> 1x1 conv blocks)"
+resnet101,False,vision,True,29M,"cnn,image-classification,residuals,resnet-variant","Bottlenecks with only conv2d (1x1 conv -> 3x3 conv -> 1x1 conv blocks)"
+squeezenet1_0,False,vision,True,1.25M,"cnn,image-classification,mobile,parallel-layers","Parallel conv2d (1x1 conv to compress -> (3x3 expand | 1x1 expand) -> concat)"
+wide_resnet50_2,False,vision,True,69M,"cnn,image-classification,residuals,resnet-variant","Resnet variant where model depth is decreased and width is increased."
+mobilenet_v3_small,False,vision,True,2.5M,"image-classification,cnn,mobile",N/A
+google/vit-base-patch16-224,True,hf_img_cls,False,86M,"image-classification,vision-transformer,transformer-encoder",N/A
+microsoft/resnet-50,True,hf_img_cls,False,23M,"image-classification,cnn,residuals,resnet-variant","Bottlenecks with only conv2d (1x1 conv -> 3x3 conv -> 1x1 conv blocks)"
+facebook/deit-small-distilled-patch16-224,True,hf_img_cls,False,22M,"image-classification,vision-transformer,cnn",N/A
+microsoft/beit-base-patch16-224-pt22k-ft22k,True,hf_img_cls,False,86M,"image-classification,transformer-encoder,bert-variant,vision-transformer",N/A
+nvidia/mit-b0,True,hf_img_cls,False,3.7M,"image-classification,transformer-encoder",SegFormer