Generate test-specific repro path for each TF model test. (#158)

Set TempFileSaver path directory to shark_args.repro_dir

shark/parser.py

@@ -20,9 +20,8 @@ def dir_path(path):
     if os.path.isdir(path):
         return path
     else:
-        raise argparse.ArgumentTypeError(
-            f"readable_dir:{path} is not a valid path"
-        )
+        os.mkdir(path)
+        return path
 
 
 def dir_file(path):
@@ -44,7 +43,8 @@ parser.add_argument(
 parser.add_argument(
     "--repro_dir",
     help="Directory to which module files will be saved for reproduction or debugging.",
-    default="./shark_tmp/",
+    type=dir_path,
+    default="./shark_tmp",
 )
 parser.add_argument(
     "--save_mlir",
@@ -78,7 +78,3 @@ parser.add_argument(
 )
 
 shark_args, unknown = parser.parse_known_args()
-
-if not os.path.exists(shark_args.repro_dir):
-    # Create a new directory because it does not exist
-    os.makedirs(shark_args.repro_dir)

tank/tf/hf_masked_lm/albert-base-v2_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class AlbertBaseModuleTester:
@@ -28,12 +29,21 @@ class AlbertBaseModuleTester:
         model, input, act_out = get_causal_lm_model("albert-base-v2")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"shark_tmp/albert_base_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"albert_base_v2_dynamic_{device}"
-            else:
-                repro_dir = f"albert_base_v2_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])
@@ -83,9 +93,9 @@ class AlbertBaseModuleTest(unittest.TestCase):
         self.module_tester.save_vmfb = pytestconfig.getoption("save_vmfb")
         self.module_tester.benchmark = pytestconfig.getoption("benchmark")
 
-    @pytest.mark.xfail(
-        reason="Upstream IREE issue, see https://github.com/google/iree/issues/9536"
-    )
+    #    @pytest.mark.xfail(
+    #       reason="Upstream IREE issue, see https://github.com/google/iree/issues/9536"
+    #  )
     def test_module_static_cpu(self):
         dynamic = False
         device = "cpu"

tank/tf/hf_masked_lm/bert-base-uncased_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class BertBaseUncasedModuleTester:
@@ -28,12 +29,21 @@ class BertBaseUncasedModuleTester:
         model, input, act_out = get_causal_lm_model("bert-base-uncased")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/bert_base_uncased_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"bert_base_uncased_dynamic_{device}"
-            else:
-                repro_dir = f"bert_base_uncased_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/camembert-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class CamemBertModuleTester:
@@ -28,12 +29,21 @@ class CamemBertModuleTester:
         model, input, act_out = get_causal_lm_model("camembert-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/camembert_base_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"camembert-base_dynamic_{device}"
-            else:
-                repro_dir = f"camembert-base_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/convbert-base-turkish-cased_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class ConvBertModuleTester:
@@ -30,12 +31,21 @@ class ConvBertModuleTester:
         )
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/convbert_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"convbert_base_dynamic_{device}"
-            else:
-                repro_dir = f"convbert_base_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/deberta-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class DebertaModuleTester:
@@ -27,12 +28,21 @@ class DebertaModuleTester:
         model, input, act_out = get_causal_lm_model("microsoft/deberta-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/deberta_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"deberta-base_dynamic_{device}"
-            else:
-                repro_dir = f"deberta-base_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/distilbert-base-uncased_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class DistilBertModuleTester:
@@ -27,12 +28,21 @@ class DistilBertModuleTester:
         model, input, act_out = get_causal_lm_model("distilbert-base-uncased")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/distilbert_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"distilbert_dynamic_{device}"
-            else:
-                repro_dir = f"distilbert__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/electra-small-discriminator_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class ElectraModuleTester:
@@ -29,12 +30,21 @@ class ElectraModuleTester:
         )
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/electra_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"electra_dynamic_{device}"
-            else:
-                repro_dir = f"electra__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/funnel-transformer_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class FunnelModuleTester:
@@ -27,12 +28,21 @@ class FunnelModuleTester:
         model, input, act_out = get_causal_lm_model("funnel-transformer/small")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/funnel_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"funnel_dynamic_{device}"
-            else:
-                repro_dir = f"funnel__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/layoutlm-base-uncased_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class LayoutLmModuleTester:
@@ -29,12 +30,21 @@ class LayoutLmModuleTester:
         )
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/layoutlm_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"layoutlm_dynamic_{device}"
-            else:
-                repro_dir = f"layoutlm__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/longformer-base-4096_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class LongFormerModuleTester:
@@ -29,12 +30,21 @@ class LongFormerModuleTester:
         )
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/longformer_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"longformer_dynamic_{device}"
-            else:
-                repro_dir = f"longformer__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/mobilebert-uncased_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class MobileBertModuleTester:
@@ -27,12 +28,21 @@ class MobileBertModuleTester:
         model, input, act_out = get_causal_lm_model("google/mobilebert-uncased")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/mobilebert_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"mobilebert_dynamic_{device}"
-            else:
-                repro_dir = f"mobilebert__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/mpnet-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class MpNetModuleTester:
@@ -27,12 +28,21 @@ class MpNetModuleTester:
         model, input, act_out = get_causal_lm_model("microsoft/mpnet-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/mpnet_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"mpnet_dynamic_{device}"
-            else:
-                repro_dir = f"mpnet__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/rembert_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class RemBertModuleTester:
@@ -27,12 +28,21 @@ class RemBertModuleTester:
         model, input, act_out = get_causal_lm_model("google/rembert")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/rembert_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"rembert_dynamic_{device}"
-            else:
-                repro_dir = f"rembert__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/roberta-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class RobertaModuleTester:
@@ -27,12 +28,21 @@ class RobertaModuleTester:
         model, input, act_out = get_causal_lm_model("roberta-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/roberta_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"roberta_dynamic_{device}"
-            else:
-                repro_dir = f"roberta__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/tapas-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class TapasBaseModuleTester:
@@ -27,12 +28,21 @@ class TapasBaseModuleTester:
         model, input, act_out = get_causal_lm_model("google/tapas-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/tapas_base_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"tapas-base_dynamic_{device}"
-            else:
-                repro_dir = f"tapas-base__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/tiny-random-flaubert_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class FlauBertModuleTester:
@@ -29,12 +30,21 @@ class FlauBertModuleTester:
         )
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/tiny_flaubert_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"flaubert_dynamic_{device}"
-            else:
-                repro_dir = f"flaubert__static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])

tank/tf/hf_masked_lm/xlm-roberta-base_test.py

@@ -9,6 +9,7 @@ import unittest
 import pytest
 import numpy as np
 import tempfile
+import os
 
 
 class XLMRobertaModuleTester:
@@ -27,12 +28,21 @@ class XLMRobertaModuleTester:
         model, input, act_out = get_causal_lm_model("xlm-roberta-base")
         shark_args.save_mlir = self.save_mlir
         shark_args.save_vmfb = self.save_vmfb
+
+        if (
+            shark_args.save_mlir == True
+            or shark_args.save_vmfb == True
+            or self.save_temps == True
+        ):
+            repro_path = f"./shark_tmp/xlm_roberta_tf_{dynamic}_{device}"
+            if not os.path.isdir(repro_path):
+                os.mkdir(repro_path)
+            shark_args.repro_dir = repro_path
+
         if self.save_temps == True:
-            if dynamic == True:
-                repro_dir = f"xlm_roberta_dynamic_{device}"
-            else:
-                repro_dir = f"xlm_roberta_static_{device}"
-            temp_dir = tempfile.mkdtemp(prefix=repro_dir)
+            temp_dir = tempfile.mkdtemp(
+                prefix="iree_tfs", dir=shark_args.repro_dir
+            )
             np.set_printoptions(threshold=np.inf)
             np.save(f"{temp_dir}/input1.npy", input[0])
             np.save(f"{temp_dir}/input2.npy", input[1])