github/AMD-SHARK-Studio
1
1
Fork 0
mirror of https://github.com/nod-ai/AMD-SHARK-Studio.git synced 2026-04-03 03:00:17 -04:00
Code Issues Packages Projects Releases Wiki Activity

Xfail vulkan tests and Enable MacOs test on CI (#383)

Browse Source
This commit is contained in:
Phaneesh Barwaria
2022-10-13 21:44:41 +05:30
committed by GitHub
parent 8f3f00cd99
commit 00a8ce75d1
2 changed files with 50 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
.github/workflows/test-models.yml vendored
View File

@@ -32,8 +32,6 @@ jobs:
suite: cuda
- os: MacStudio
suite: cpu
- os: MacStudio
suite: vulkan
- os: icelake
suite: vulkan
- os: icelake
@@ -104,8 +102,29 @@ jobs:
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv
gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cuda_latest.csv
- name: Validate Vulkan Models
if: matrix.suite == 'vulkan'
- name: Validate Vulkan Models MacOS
if: matrix.suite == 'vulkan' && matrix.os == 'MacStudio'
run: |
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} BENCHMARK=1 IMPORTER=1 ./setup_venv.sh
source shark.venv/bin/activate
echo "VULKAN SDK PATH wo setup: $VULKAN_SDK"
cd /Users/anush/VulkanSDK/1.3.224.1/
source setup-env.sh
cd $GITHUB_WORKSPACE
echo "VULKAN SDK PATH with setup: $VULKAN_SDK"
echo $PATH
pip list | grep -E "torch|iree"
pip uninstall -y torch iree-compiler iree-runtime
pip install https://download.pytorch.org/whl/nightly/cpu/torch-1.14.0.dev20221010-cp310-none-macosx_11_0_arm64.whl
pip install https://github.com/llvm/torch-mlir/releases/download/oneshot-20221011.55/torch_mlir-20221011.55-cp310-cp310-macosx_11_0_universal2.whl
pip install https://github.com/nod-ai/SHARK-Runtime/releases/download/candidate-20221011.179/iree_compiler-20221011.179-cp310-cp310-macosx_11_0_universal2.whl
pip install https://github.com/nod-ai/SHARK-Runtime/releases/download/candidate-20221011.179/iree_runtime-20221011.179-cp310-cp310-macosx_11_0_universal2.whl
pip list | grep -E "torch|iree"
pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush" tank/test_models.py -k vulkan
- name: Validate Vulkan Models (icelake, a100, ubuntu)
if: matrix.suite == 'vulkan' && matrix.os != 'MacStudio'
run: |
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} BENCHMARK=1 IMPORTER=1 ./setup_venv.sh

27
tank/test_models.py
View File

@@ -365,6 +365,33 @@ class SharkModuleTest(unittest.TestCase):
)
if config["model_name"] == "hf-internal-testing/tiny-random-flaubert":
pytest.xfail(reason="Transformers API mismatch")
if config["model_name"] == "alexnet" and device in ["metal", "vulkan"]:
pytest.xfail(reason="Assertion Error: Zeros Output")
if (
config["model_name"] == "camembert-base"
and dynamic == False
and device in ["metal", "vulkan"]
):
pytest.xfail(
reason="chlo.broadcast_compare failed to satify constraint"
)
if (
config["model_name"] == "roberta-base"
and dynamic == False
and device in ["metal", "vulkan"]
):
pytest.xfail(
reason="chlo.broadcast_compare failed to satify constraint"
)
if config["model_name"] in [
"microsoft/MiniLM-L12-H384-uncased",
"wide_resnet50_2",
"resnet50",
"resnet18",
"resnet101",
"microsoft/resnet-50",
] and device in ["metal", "vulkan"]:
pytest.xfail(reason="Vulkan Numerical Error (mostly conv)")
if config["framework"] == "tf" and dynamic == True:
pytest.skip(
reason="Dynamic shapes not supported for this framework."