diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3c042657..b163f74f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -108,6 +108,7 @@ jobs: - name: Install dependencies run: | + echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV python -m pip install --upgrade pip python -m pip install flake8 pytest toml if [ -f requirements.txt ]; then pip install -r requirements.txt -f https://llvm.github.io/torch-mlir/package-index/ -f https://nod-ai.github.io/SHARK-Runtime/pip-release-links.html; fi @@ -131,7 +132,7 @@ jobs: pip install ./wheelhouse/nodai* # Validate the Models /bin/bash "$GITHUB_WORKSPACE/build_tools/populate_sharktank_ci.sh" - pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="./gen_shark_tank/" tank/test_models.py | + pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="./gen_shark_tank/" -k "not metal" | tail -n 1 | tee -a pytest_results.txt if !(grep -Fxq " failed" pytest_results.txt) @@ -154,6 +155,6 @@ jobs: # Install the built wheel pip install ./wheelhouse/nodai* # Validate the Models - pytest --ci --ci_sha=${SHORT_SHA} tank/test_models.py | + pytest --ci --ci_sha=${SHORT_SHA} -k "not metal" | tail -n 1 | tee -a pytest_results.txt diff --git a/.github/workflows/test-models.yml b/.github/workflows/test-models.yml index c477b143..146f37e2 100644 --- a/.github/workflows/test-models.yml +++ b/.github/workflows/test-models.yml @@ -6,8 +6,14 @@ name: Validate Models on Shark Runtime on: push: branches: [ main ] + paths-ignore: + - '**.md' + - 'shark/examples/**' pull_request: branches: [ main ] + paths-ignore: + - '**.md' + - 'shark/examples/**' workflow_dispatch: # Ensure that only a single job or workflow using the same @@ -117,9 +123,9 @@ jobs: PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh source shark.venv/bin/activate export DYLD_LIBRARY_PATH=/usr/local/lib/ - cd $GITHUB_WORKSPACE + echo $PATH pip list | grep -E "torch|iree" - pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" tank/test_models.py -k vulkan --update_tank + pytest -s --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" tank/test_models.py -k vulkan --update_tank - name: Validate Vulkan Models (a100) if: matrix.suite == 'vulkan' && matrix.os != 'MacStudio' diff --git a/.gitignore b/.gitignore index a4aa870d..0f61f8c3 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,7 @@ cython_debug/ shark_tmp/ *.vmfb .use-iree +tank/dict_configs.py # ORT related artefacts cache_models/ diff --git a/pytest.ini b/pytest.ini index 3bda9959..3634f600 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] addopts = --verbose -p no:warnings -norecursedirs = inference tank/tflite +norecursedirs = inference tank/tflite examples benchmarks shark diff --git a/shark/parser.py b/shark/parser.py index ff7fe95b..52569fee 100644 --- a/shark/parser.py +++ b/shark/parser.py @@ -108,7 +108,7 @@ parser.add_argument( parser.add_argument( "--enable_conv_transform", default=False, - action="store", + action="store_true", help="Enables the --iree-flow-enable-conv-nchw-to-nhwc-transform flag.", ) diff --git a/shark/shark_benchmark_runner.py b/shark/shark_benchmark_runner.py index 192dfdd3..533b9beb 100644 --- a/shark/shark_benchmark_runner.py +++ b/shark/shark_benchmark_runner.py @@ -100,6 +100,7 @@ class SharkBenchmarkRunner(SharkRunner): def benchmark_frontend(self, modelname): if self.mlir_dialect in ["linalg", "torch"]: return self.benchmark_torch(modelname) + elif self.mlir_dialect in ["mhlo", "tf"]: return self.benchmark_tf(modelname) @@ -138,9 +139,21 @@ class SharkBenchmarkRunner(SharkRunner): def benchmark_tf(self, modelname): import tensorflow as tf + + visible_default = tf.config.list_physical_devices("GPU") + try: + tf.config.set_visible_devices([], "GPU") + visible_devices = tf.config.get_visible_devices() + for device in visible_devices: + assert device.device_type != "GPU" + except: + # Invalid device or cannot modify virtual devices once initialized. + pass + from tank.model_utils_tf import get_tf_model - tf_device = "/GPU:0" if self.device == "cuda" else "/CPU:0" + # tf_device = "/GPU:0" if self.device == "cuda" else "/CPU:0" + tf_device = "/CPU:0" with tf.device(tf_device): model, input, = get_tf_model( modelname diff --git a/shark/shark_downloader.py b/shark/shark_downloader.py index dcddb9c8..9a0a8078 100644 --- a/shark/shark_downloader.py +++ b/shark/shark_downloader.py @@ -138,7 +138,11 @@ def download_model( model_dir = os.path.join(WORKDIR, model_dir_name) full_gs_url = tank_url.rstrip("/") + "/" + model_dir_name - if not check_dir_exists( + if shark_args.update_tank == True: + print(f"Updating artifacts for model {model_name}...") + download_public_file(full_gs_url, model_dir) + + elif not check_dir_exists( model_dir_name, frontend=frontend, dynamic=dyn_str ): print(f"Downloading artifacts for model {model_name}...") @@ -162,13 +166,9 @@ def download_model( np.load(os.path.join(model_dir, "upstream_hash.npy")) ) if local_hash != upstream_hash: - if shark_args.update_tank == True: - print(f"Updating artifacts for model {model_name}...") - download_public_file(full_gs_url, WORKDIR) - else: - print( - "Hash does not match upstream in gs://shark_tank/. If you are using SHARK Downloader with locally generated artifacts, this is working as intended." - ) + print( + "Hash does not match upstream in gs://shark_tank/latest. If you want to use locally generated artifacts, this is working as intended. Otherwise, run with --update_tank." + ) model_dir = os.path.join(WORKDIR, model_dir_name) tuned_str = "" if tuned is None else "_" + tuned diff --git a/tank/all_models.csv b/tank/all_models.csv index a85cbc92..77f199c3 100644 --- a/tank/all_models.csv +++ b/tank/all_models.csv @@ -1,35 +1,34 @@ -resnet50,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc -albert-base-v2,mhlo,tf,1e-2,1e-2,default,None -roberta-base,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc -bert-base-uncased,mhlo,tf,1e-2,1e-3,default,None -camembert-base,mhlo,tf,1e-2,1e-3,default,None -dbmdz/convbert-base-turkish-cased,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc -distilbert-base-uncased,mhlo,tf,1e-2,1e-3,default,None -facebook/convnext-tiny-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc, -funnel-transformer/small,mhlo,tf,1e-2,1e-3,default,None -google/electra-small-discriminator,mhlo,tf,1e-2,1e-3,default,None -google/mobilebert-uncased,mhlo,tf,1e-2,1e-3,default,None -google/vit-base-patch16-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc -hf-internal-testing/tiny-random-flaubert,mhlo,tf,1e-2,1e-3,default,None -microsoft/MiniLM-L12-H384-uncased,mhlo,tf,1e-2,1e-3,tf_hf,None -microsoft/layoutlm-base-uncased,mhlo,tf,1e-2,1e-3,default,None -microsoft/mpnet-base,mhlo,tf,1e-2,1e-2,default,None -albert-base-v2,linalg,torch,1e-2,1e-3,default,None -alexnet,linalg,torch,1e-2,1e-3,default,None -bert-base-cased,linalg,torch,1e-2,1e-3,default,None -bert-base-uncased,linalg,torch,1e-2,1e-3,default,None -facebook/deit-small-distilled-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -google/vit-base-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -microsoft/beit-base-patch16-224-pt22k-ft22k,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -microsoft/MiniLM-L12-H384-uncased,linalg,torch,1e-2,1e-3,default,None -microsoft/resnet-50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -google/mobilebert-uncased,linalg,torch,1e-2,1e-3,default,None -mobilenet_v3_small,linalg,torch,1e-1,1e-2,default,nhcw-nhwc -nvidia/mit-b0,linalg,torch,1e-2,1e-3,default,None -resnet101,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -resnet18,linalg,torch,1e-2,1e-3,default,None -resnet50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -squeezenet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -wide_resnet50_2,linalg,torch,1e-2,1e-3,default,nhcw-nhwc -efficientnet-v2-s,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc -mnasnet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc +resnet50,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error: mostly conv" +albert-base-v2,mhlo,tf,1e-2,1e-2,default,None,False,False,False,"" +roberta-base,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc,False,False,False,"" +bert-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,"" +camembert-base,mhlo,tf,1e-2,1e-3,default,None,False,False,False,"" +dbmdz/convbert-base-turkish-cased,mhlo,tf,1e-2,1e-3,default,nhcw-nhwc,True,True,True,"https://github.com/iree-org/iree/issues/9971" +distilbert-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,"" +facebook/convnext-tiny-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc,True,True,True,"https://github.com/nod-ai/SHARK/issues/311 & https://github.com/nod-ai/SHARK/issues/342" +funnel-transformer/small,mhlo,tf,1e-2,1e-3,default,None,True,True,True,"https://github.com/nod-ai/SHARK/issues/201" +google/electra-small-discriminator,mhlo,tf,1e-2,1e-3,default,None,False,False,False,"" +google/mobilebert-uncased,mhlo,tf,1e-2,1e-3,default,None,True,False,False,"Fails during iree-compile." +google/vit-base-patch16-224,mhlo,tf,1e-2,1e-3,tf_vit,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)" +microsoft/MiniLM-L12-H384-uncased,mhlo,tf,1e-2,1e-3,tf_hf,None,True,False,False,"Fails during iree-compile." +microsoft/layoutlm-base-uncased,mhlo,tf,1e-2,1e-3,default,None,False,False,False,"" +microsoft/mpnet-base,mhlo,tf,1e-2,1e-2,default,None,False,False,False,"" +albert-base-v2,linalg,torch,1e-2,1e-3,default,None,True,True,True,"issue with aten.tanh in torch-mlir" +alexnet,linalg,torch,1e-2,1e-3,default,None,False,False,True,"Assertion Error: Zeros Output" +bert-base-cased,linalg,torch,1e-2,1e-3,default,None,False,False,False,"" +bert-base-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,False,"" +facebook/deit-small-distilled-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"Fails during iree-compile." +google/vit-base-patch16-224,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"https://github.com/nod-ai/SHARK/issues/311" +microsoft/beit-base-patch16-224-pt22k-ft22k,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,True,False,"https://github.com/nod-ai/SHARK/issues/390" +microsoft/MiniLM-L12-H384-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,True,"" +microsoft/resnet-50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)" +google/mobilebert-uncased,linalg,torch,1e-2,1e-3,default,None,False,False,True,"https://github.com/nod-ai/SHARK/issues/344" +mobilenet_v3_small,linalg,torch,1e-1,1e-2,default,nhcw-nhwc,False,True,True,"https://github.com/nod-ai/SHARK/issues/388" +nvidia/mit-b0,linalg,torch,1e-2,1e-3,default,None,True,True,True,"https://github.com/nod-ai/SHARK/issues/343" +resnet101,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)" +resnet18,linalg,torch,1e-2,1e-3,default,None,True,True,True,"" +resnet50,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)" +squeezenet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/388" +wide_resnet50_2,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"Vulkan Numerical Error (mostly conv)" +efficientnet-v2-s,mhlo,tf,1e-02,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/575" +mnasnet1_0,linalg,torch,1e-2,1e-3,default,nhcw-nhwc,False,False,True,"https://github.com/nod-ai/SHARK/issues/388" diff --git a/tank/test_models.py b/tank/test_models.py index 07a41334..fdf90e4a 100644 --- a/tank/test_models.py +++ b/tank/test_models.py @@ -39,6 +39,10 @@ def load_csv_and_convert(filename, gen=False): "atol": float(row[4]), "out_type": row[5], "flags": row[6], + "xfail_cpu": row[7], + "xfail_cuda": row[8], + "xfail_vkm": row[9], + "xfail_reason": row[10], } ) # This is a pytest workaround @@ -168,16 +172,32 @@ class SharkModuleTester: rtol=self.config["rtol"], atol=self.config["atol"], ) - except AssertionError: + except AssertionError as msg: if any([self.ci, self.save_repro, self.save_fails]) == True: self.save_reproducers() if self.ci == True: self.upload_repro() if self.benchmark == True: + # p = multiprocessing.Process( + # target=self.benchmark_module, + # args=(shark_module, inputs, dynamic, device), + # ) + # p.start() + # p.join() self.benchmark_module(shark_module, inputs, dynamic, device) - raise + print(msg) + pytest.xfail(reason="Numerics Issue") if self.benchmark == True: + # We must create a new process each time we benchmark a model to allow + # for Tensorflow to release GPU resources. Using the same process to + # benchmark multiple models leads to OOM. + # p = multiprocessing.Process( + # target=self.benchmark_module, + # args=(shark_module, inputs, dynamic, device), + # ) + # p.start() + # p.join() self.benchmark_module(shark_module, inputs, dynamic, device) if self.save_repro == True: @@ -233,16 +253,6 @@ class SharkModuleTester: return expected, logits -def run_test(module_tester, dynamic, device): - tempdir = tempfile.TemporaryDirectory( - prefix=module_tester.tmp_prefix, dir="./shark_tmp/" - ) - module_tester.temp_dir = tempdir.name - - with ireec.tools.TempFileSaver(tempdir.name): - module_tester.create_and_check_module(dynamic, device) - - class SharkModuleTest(unittest.TestCase): @pytest.fixture(autouse=True) def configure(self, pytestconfig): @@ -273,15 +283,17 @@ class SharkModuleTest(unittest.TestCase): "update_tank" ) self.module_tester.tank_url = self.pytestconfig.getoption("tank_url") - if config["model_name"] == "efficientnet-v2-s" and device in [ - "metal", - "vulkan", - ]: - pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/575") - if config[ - "model_name" - ] == "google/vit-base-patch16-224" and device in ["cuda"]: - pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/311") + + if config["xfail_cpu"] == "True" and device == "cpu": + pytest.xfail(reason=config["xfail_reason"]) + + if config["xfail_cuda"] == "True" and device == "cuda": + pytest.xfail(reason=config["xfail_reason"]) + + if config["xfail_vkm"] == "True" and device in ["metal", "vulkan"]: + pytest.xfail(reason=config["xfail_reason"]) + + # Special cases that need to be marked. if config["model_name"] == "resnet50" and device in [ "metal", "vulkan", @@ -291,78 +303,6 @@ class SharkModuleTest(unittest.TestCase): pytest.xfail( reason="M2: Assert Error & M1: CompilerToolError" ) - if config[ - "model_name" - ] == "dbmdz/convbert-base-turkish-cased" and device in [ - "metal", - "vulkan", - ]: - pytest.xfail( - reason="Issue: https://github.com/iree-org/iree/issues/9971" - ) - if config["model_name"] == "facebook/convnext-tiny-224" and device in [ - "cuda", - "metal", - "vulkan", - ]: - pytest.xfail( - reason="https://github.com/nod-ai/SHARK/issues/311, https://github.com/nod-ai/SHARK/issues/342" - ) - if config["model_name"] == "funnel-transformer/small" and device in [ - "cuda", - "metal", - "vulkan", - ]: - pytest.xfail( - reason="failing in the iree-compiler passes, see https://github.com/nod-ai/SHARK/issues/201" - ) - if config["model_name"] == "nvidia/mit-b0": - pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/343") - if ( - config["model_name"] == "google/mobilebert-uncased" - and device in ["metal", "vulkan"] - and config["framework"] == "torch" - ): - pytest.xfail( - reason="Numerics issues -- https://github.com/nod-ai/SHARK/issues/344" - ) - if ( - config["model_name"] == "facebook/deit-small-distilled-patch16-224" - and device == "cuda" - ): - pytest.xfail( - reason="Fails during iree-compile without reporting diagnostics." - ) - if ( - config["model_name"] - == "microsoft/beit-base-patch16-224-pt22k-ft22k" - and device == "cuda" - ): - pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/390") - if config["model_name"] == "squeezenet1_0" and device in [ - "metal", - "vulkan", - ]: - pytest.xfail( - reason="Numerics Issues: https://github.com/nod-ai/SHARK/issues/388" - ) - if config["model_name"] == "mobilenet_v3_small" and device not in [ - "cpu" - ]: - pytest.xfail( - reason="Numerics Issues: https://github.com/nod-ai/SHARK/issues/388" - ) - if config["model_name"] == "mnasnet1_0" and device not in [ - "cpu", - "cuda", - ]: - pytest.xfail( - reason="Numerics Issues: https://github.com/nod-ai/SHARK/issues/388" - ) - if config["model_name"] == "hf-internal-testing/tiny-random-flaubert": - pytest.xfail(reason="Transformers API mismatch") - if config["model_name"] == "alexnet" and device in ["metal", "vulkan"]: - pytest.xfail(reason="Assertion Error: Zeros Output") if ( config["model_name"] == "camembert-base" and dynamic == False @@ -379,19 +319,6 @@ class SharkModuleTest(unittest.TestCase): pytest.xfail( reason="chlo.broadcast_compare failed to satify constraint" ) - if config["model_name"] in [ - "microsoft/MiniLM-L12-H384-uncased", - "wide_resnet50_2", - "resnet50", - "resnet18", - "resnet101", - "microsoft/resnet-50", - ] and device in ["metal", "vulkan"]: - pytest.xfail(reason="Vulkan Numerical Error (mostly conv)") - if config[ - "model_name" - ] == "dbmdz/convbert-base-turkish-cased" and device in ["cuda", "cpu"]: - pytest.xfail(reason="https://github.com/nod-ai/SHARK/issues/463") if ( config["model_name"] in [ @@ -414,11 +341,6 @@ class SharkModuleTest(unittest.TestCase): pytest.xfail( reason="Numerics issues: https://github.com/nod-ai/SHARK/issues/476" ) - if config["framework"] == "tf" and dynamic == True: - pytest.skip( - reason="Dynamic shapes not supported for this framework." - ) - safe_name = ( f"{config['model_name']}_{config['framework']}_{dynamic}_{device}" ) @@ -427,11 +349,10 @@ class SharkModuleTest(unittest.TestCase): if not os.path.isdir("./shark_tmp/"): os.mkdir("./shark_tmp/") - # We must create a new process each time we benchmark a model to allow - # for Tensorflow to release GPU resources. Using the same process to - # benchmark multiple models leads to OOM. - p = multiprocessing.Process( - target=run_test, args=(self.module_tester, dynamic, device) + tempdir = tempfile.TemporaryDirectory( + prefix=self.module_tester.tmp_prefix, dir="./shark_tmp/" ) - p.start() - p.join() + self.module_tester.temp_dir = tempdir.name + + with ireec.tools.TempFileSaver(tempdir.name): + self.module_tester.create_and_check_module(dynamic, device)