github/SHARK-Studio
1
1
Fork 0
mirror of https://github.com/nod-ai/SHARK-Studio.git synced 2026-01-09 22:07:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Enable CPU benchmarks on test-models workflows. (#299)

Browse Source 
* Update test-models.yml

* Update README.md
This commit is contained in:
Ean Garvey
2022-09-07 01:22:58 -05:00
committed by GitHub
parent 3824d37d27
commit d453f2e49d
2 changed files with 12 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/test-models.yml vendored
View File

@@ -52,13 +52,13 @@ jobs:
echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
- name: Set up Python Version File ${{ matrix.python-version }}
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest'
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest' || matrix.os == 'icelake'
run: |
# See https://github.com/actions/setup-python/issues/433
echo ${{ matrix.python-version }} >> $GITHUB_WORKSPACE/.python-version
- name: Set up Python ${{ matrix.python-version }}
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest'
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest' || matrix.os == 'icelake'
uses: actions/setup-python@v4
with:
python-version: '${{ matrix.python-version }}'
@@ -90,7 +90,9 @@ jobs:
cd $GITHUB_WORKSPACE
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
source shark.venv/bin/activate
pytest -k 'cpu' --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
pytest --benchmark tank/ -k cpu
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv
gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cpu_latest.csv
- name: Validate GPU Models
if: matrix.suite == 'gpu'
@@ -100,6 +102,7 @@ jobs:
source shark.venv/bin/activate
pytest --benchmark -k "gpu" --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_gpu_${SHORT_SHA}.csv
gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_gpu_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_gpu_latest.csv
- name: Validate Vulkan Models
if: matrix.suite == 'vulkan'

8
README.md
View File

@@ -99,7 +99,7 @@ pytest tank/tf/hf_masked_lm/albert-base-v2_test.py::AlbertBaseModuleTest::test_m
<details>
<summary>Testing</summary>
<summary>Testing and Benchmarks</summary>
### Run all model tests on CPU/GPU/VULKAN/Metal
```shell
@@ -122,7 +122,11 @@ pytest tank/<MODEL_NAME> -k "keyword"
### Run benchmarks on SHARK tank pytests and generate bench_results.csv with results.
(requires source installation with `IMPORTER=1 ./setup_venv.sh`)
Note: Latest benchmarks on our canonical machines can be found here:
https://storage.googleapis.com/shark-public/builder/bench_results/latest/bench_results_cpu_latest.csv
https://storage.googleapis.com/shark-public/builder/bench_results/latest/bench_results_gpu_latest.csv
(the following requires source installation with `IMPORTER=1 ./setup_venv.sh`)
```shell
pytest --benchmark tank