Update expected failures in pytest suite. (#574)

tank/all_models.csv

@@ -1,5 +1,5 @@
-resnet50,mhlo,tf,1e-02,1e-3,default
-albert-base-v2,mhlo,tf,1e-02,1e-3,default
+resnet50,mhlo,tf,1e-2,1e-3,default
+albert-base-v2,mhlo,tf,1e-2,1e-2,default
 roberta-base,mhlo,tf,1e-02,1e-3,default
 bert-base-uncased,mhlo,tf,1e-2,1e-3,default
 camembert-base,mhlo,tf,1e-2,1e-3,default
@@ -13,19 +13,18 @@ google/vit-base-patch16-224,mhlo,tf,1e-2,1e-3,tf_vit
 hf-internal-testing/tiny-random-flaubert,mhlo,tf,1e-2,1e-3,default
 microsoft/MiniLM-L12-H384-uncased,mhlo,tf,1e-2,1e-3,tf_hf
 microsoft/layoutlm-base-uncased,mhlo,tf,1e-2,1e-3,default
-microsoft/mpnet-base,mhlo,tf,1e-2,1e-3,default
+microsoft/mpnet-base,mhlo,tf,1e-2,1e-2,default
 albert-base-v2,linalg,torch,1e-2,1e-3,default
 alexnet,linalg,torch,1e-2,1e-3,default
 bert-base-cased,linalg,torch,1e-2,1e-3,default
 bert-base-uncased,linalg,torch,1e-2,1e-3,default
-distilbert-base-uncased,linalg,torch,1e-2,1e-3,default
 facebook/deit-small-distilled-patch16-224,linalg,torch,1e-2,1e-3,default
 google/vit-base-patch16-224,linalg,torch,1e-2,1e-3,default
 microsoft/beit-base-patch16-224-pt22k-ft22k,linalg,torch,1e-2,1e-3,default
 microsoft/MiniLM-L12-H384-uncased,linalg,torch,1e-2,1e-3,default
 microsoft/resnet-50,linalg,torch,1e-2,1e-3,default
 google/mobilebert-uncased,linalg,torch,1e-2,1e-3,default
-mobilenet_v3_small,linalg,torch,1e-2,1e-3,default
+mobilenet_v3_small,linalg,torch,1e-1,1e-2,default
 nvidia/mit-b0,linalg,torch,1e-2,1e-3,default
 resnet101,linalg,torch,1e-2,1e-3,default
 resnet18,linalg,torch,1e-2,1e-3,default

tank/test_models.py

@@ -266,19 +266,14 @@ class SharkModuleTest(unittest.TestCase):
             "update_tank"
         )
         self.module_tester.tank_url = self.pytestconfig.getoption("tank_url")
-        if (
-            config["model_name"] == "distilbert-base-uncased"
-            and config["framework"] == "torch"
-        ):
-            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/354")
-        if config["model_name"] == "facebook/convnext-tiny-224" and device in [
-            "cuda",
-            "cpu",
+        if config["model_name"] == "efficientnet-v2-s" and device in [
+            "metal",
+            "vulkan",
         ]:
-            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
+            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/575")
         if config[
             "model_name"
-        ] == "google/vit-base-patch16-224" and device in ["cuda", "cpu"]:
+        ] == "google/vit-base-patch16-224" and device in ["cuda"]:
             pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
         if config["model_name"] == "resnet50" and device in [
             "metal",
@@ -289,8 +284,6 @@ class SharkModuleTest(unittest.TestCase):
                     pytest.xfail(
                         reason="M2: Assert Error & M1: CompilerToolError"
                     )
-        if config["model_name"] == "google/rembert":
-            pytest.skip(reason="Model too large to convert.")
         if config[
             "model_name"
         ] == "dbmdz/convbert-base-turkish-cased" and device in [
@@ -301,7 +294,6 @@ class SharkModuleTest(unittest.TestCase):
                 reason="Issue: https://github.com/iree-org/iree/issues/9971"
             )
         if config["model_name"] == "facebook/convnext-tiny-224" and device in [
-            "cpu",
             "cuda",
             "metal",
             "vulkan",
@@ -310,7 +302,6 @@ class SharkModuleTest(unittest.TestCase):
                 reason="https://github.com/nod-ai/SHARK/issues/311, https://github.com/nod-ai/SHARK/issues/342"
             )
         if config["model_name"] == "funnel-transformer/small" and device in [
-            "cpu",
             "cuda",
             "metal",
             "vulkan",
@@ -318,13 +309,6 @@ class SharkModuleTest(unittest.TestCase):
             pytest.xfail(
                 reason="failing in the iree-compiler passes, see https://github.com/nod-ai/SHARK/issues/201"
             )
-        if (
-            config["model_name"] == "google/vit-base-patch16-224"
-            and device == "cuda"
-        ):
-            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311")
-        if config["model_name"] == "microsoft/mpnet-base":
-            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/203")
         if config["model_name"] == "nvidia/mit-b0":
             pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/343")
         if (
@@ -335,12 +319,6 @@ class SharkModuleTest(unittest.TestCase):
             pytest.xfail(
                 reason="Numerics issues -- https://github.com/nod-ai/SHARK/issues/344"
             )
-        if (
-            config["model_name"] == "google/mobilebert-uncased"
-            and device == "cpu"
-            and config["framework"] == "tf"
-        ):
-            pytest.xfail(reason="Fails during iree-compile.")
         if (
             config["model_name"] == "facebook/deit-small-distilled-patch16-224"
             and device == "cuda"
@@ -355,7 +333,6 @@ class SharkModuleTest(unittest.TestCase):
         ):
             pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/390")
         if config["model_name"] == "squeezenet1_0" and device in [
-            "cpu",
             "metal",
             "vulkan",
         ]:
@@ -398,11 +375,6 @@ class SharkModuleTest(unittest.TestCase):
             "microsoft/resnet-50",
         ] and device in ["metal", "vulkan"]:
             pytest.xfail(reason="Vulkan Numerical Error (mostly conv)")
-        if config["model_name"] == "mobilenet_v3_small" and device in [
-            "cuda",
-            "cpu",
-        ]:
-            pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/424")
         if config[
             "model_name"
         ] == "dbmdz/convbert-base-turkish-cased" and device in ["cuda", "cpu"]:
@@ -429,44 +401,6 @@ class SharkModuleTest(unittest.TestCase):
             pytest.xfail(
                 reason="Numerics issues: https://github.com/nod-ai/SHARK/issues/476"
             )
-        if (
-            config["model_name"] == "microsoft/MiniLM-L12-H384-uncased"
-            and device == "cpu"
-        ):
-            pytest.xfail(reason="Fails during iree-compile")
-        if (
-            config["model_name"] in ["albert-base-v2"]
-            and device == "cpu"
-            and dynamic == False
-            and config["framework"] == "tf"
-        ):
-            pytest.xfail(
-                reason="Numerics issues: https://github.com/nod-ai/SHARK/issues/489"
-            )
-        if (
-            config["model_name"]
-            in [
-                "microsoft/resnet-50",
-                "alexnet",
-                "resnet101",
-                "resnet18",
-                "resnet50",
-                "wide_resnet50_2",
-            ]
-            and device
-            in [
-                "cpu",
-                "cuda",
-            ]
-            and config["framework"] == "torch"
-        ):
-            pytest.xfail(
-                reason="tensor dimension issue: https://github.com/nod-ai/SHARK/issues/511"
-            )
-        if config["model_name"] == "squeezenet1_0" and device == "cuda":
-            pytest.xfail(
-                reason="tensor dimension issue: https://github.com/nod-ai/SHARK/issues/511"
-            )
         if config["framework"] == "tf" and dynamic == True:
             pytest.skip(
                 reason="Dynamic shapes not supported for this framework."
@@ -488,3 +422,4 @@ class SharkModuleTest(unittest.TestCase):
         )
         p.start()
         p.join()
+        assert not p.exitcode