github/AMD-SHARK-Studio
1
1
Fork 0
mirror of https://github.com/nod-ai/AMD-SHARK-Studio.git synced 2026-04-03 03:00:17 -04:00
Code Issues Packages Projects Releases Wiki Activity

Include SHA in path to failure reproducers. Add --save_fails option. (#352)

Browse Source
This commit is contained in:
Ean Garvey
2022-09-21 17:55:06 -05:00
committed by GitHub
parent e9ad4b9fc4
commit c0f3a09a40
3 changed files with 23 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/test-models.yml vendored
View File

@@ -90,7 +90,7 @@ jobs:
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
source shark.venv/bin/activate
pytest --benchmark --ci tank/test_models.py -k cpu
pytest --benchmark --ci --ci_sha=${SHORT_SHA} tank/test_models.py -k cpu
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv
gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cpu_latest.csv
@@ -100,7 +100,7 @@ jobs:
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
source shark.venv/bin/activate
pytest --benchmark --ci tank/test_models.py -k "cuda" --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
pytest --benchmark --ci --ci_sha=${SHORT_SHA} tank/test_models.py -k cuda
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv
gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cuda_latest.csv
@@ -110,4 +110,4 @@ jobs:
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
source shark.venv/bin/activate
pytest --ci tank/test_models.py -k 'vulkan' --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
pytest --ci --ci_sha=${SHORT_SHA} tank/test_models.py -k 'vulkan' --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py

14
conftest.py
View File

@@ -24,9 +24,21 @@ def pytest_addoption(parser):
default="False",
help="Pass option to save reproduction artifacts to SHARK/shark_tmp/test_case/",
)
parser.addoption(
"--save_fails",
action="store_true",
default="False",
help="Save reproduction artifacts for a test case only if it fails. Default is False.",
)
parser.addoption(
"--ci",
action="store_true",
default="False",
help="Enables uploading of reproduction artifacts upon test case failure during iree-compile or validation.",
help="Enables uploading of reproduction artifacts upon test case failure during iree-compile or validation. Must be passed with --ci_sha option ",
)
parser.addoption(
"--ci_sha",
action="store",
default="None",
help="Passes the github SHA of the CI workflow to include in google storage directory for reproduction artifacts.",
)

10
tank/test_models.py
View File

@@ -156,10 +156,10 @@ class SharkModuleTester:
try:
shark_module.compile()
except:
if any([self.ci, self.save_repro]) == True:
if any([self.ci, self.save_repro, self.save_fails]) == True:
self.save_reproducers()
if self.ci == True:
self.upload_repro()
self.upload_repro(self.ci_sha)
raise
result = shark_module.forward(inputs)
@@ -172,7 +172,7 @@ class SharkModuleTester:
atol=self.config["atol"],
)
except AssertionError:
if any([self.ci, self.save_repro]) == True:
if any([self.ci, self.save_repro, self.save_fails]) == True:
self.save_reproducers()
if self.ci == True:
self.upload_repro()
@@ -250,11 +250,15 @@ class SharkModuleTest(unittest.TestCase):
self.module_tester.save_repro = self.pytestconfig.getoption(
"save_repro"
)
self.module_tester.save_fails = self.pytestconfig.getoption(
"save_fails"
)
self.module_tester.onnx_bench = self.pytestconfig.getoption(
"onnx_bench"
)
self.module_tester.tf32 = self.pytestconfig.getoption("tf32")
self.module_tester.ci = self.pytestconfig.getoption("ci")
self.module_tester.ci_sha = self.pytestconfig.getoption("ci_sha")
if (
config["model_name"] == "facebook/convnext-tiny-224"
and device == "cuda"