mirror of
https://github.com/nod-ai/SHARK-Studio.git
synced 2026-01-10 14:27:58 -05:00
3514822cac
* Add ONNX env var flags for venv setup. * Setup arguments for ONNX benchmarking via pytest. * Enable ONNX benchmarking on MiniLM via pytest (experimental) * Fix sequence lengths to 128 for TF model creation and fix issue with benchmarks. * Disable CI CPU benchmarks on A100, change some default args. * add xfails for roberta TF model tests on GPU.
105 lines
3.8 KiB
YAML
105 lines
3.8 KiB
YAML
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
|
|
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
|
|
|
name: Validate Models on Shark Runtime
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build-validate:
|
|
strategy:
|
|
fail-fast: true
|
|
matrix:
|
|
os: [a100, MacStudio, ubuntu-latest]
|
|
suite: [cpu,gpu,vulkan]
|
|
python-version: ["3.10"]
|
|
include:
|
|
- os: ubuntu-latest
|
|
suite: lint
|
|
exclude:
|
|
- os: ubuntu-latest
|
|
suite: vulkan
|
|
- os: ubuntu-latest
|
|
suite: gpu
|
|
- os: ubuntu-latest
|
|
suite: cpu
|
|
- os: MacStudio
|
|
suite: gpu
|
|
- os: MacStudio
|
|
suite: cpu
|
|
- os: MacStudio
|
|
suite: vulkan
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Set Environment Variables
|
|
run: |
|
|
echo "SHORT_SHA=`git rev-parse --short=4 HEAD`" >> $GITHUB_ENV
|
|
echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
|
|
|
|
- name: Set up Python Version File ${{ matrix.python-version }}
|
|
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest'
|
|
run: |
|
|
# See https://github.com/actions/setup-python/issues/433
|
|
echo ${{ matrix.python-version }} >> $GITHUB_WORKSPACE/.python-version
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
if: matrix.os == 'a100' || matrix.os == 'ubuntu-latest'
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '${{ matrix.python-version }}'
|
|
#cache: 'pip'
|
|
#cache-dependency-path: |
|
|
# **/requirements-importer.txt
|
|
# **/requirements.txt
|
|
|
|
- name: Install dependencies
|
|
if: matrix.suite == 'lint'
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install flake8 pytest toml black
|
|
|
|
- name: Lint with flake8
|
|
if: matrix.suite == 'lint'
|
|
run: |
|
|
# black format check
|
|
black --version
|
|
black --line-length 79 --check .
|
|
# stop the build if there are Python syntax errors or undefined names
|
|
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude lit.cfg.py
|
|
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
|
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude lit.cfg.py
|
|
|
|
- name: Validate CPU Models
|
|
if: matrix.suite == 'cpu'
|
|
run: |
|
|
cd $GITHUB_WORKSPACE
|
|
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
|
|
source shark.venv/bin/activate
|
|
pytest -k 'cpu' --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
|
|
|
|
- name: Validate GPU Models
|
|
if: matrix.suite == 'gpu'
|
|
run: |
|
|
cd $GITHUB_WORKSPACE
|
|
PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
|
|
source shark.venv/bin/activate
|
|
pytest --benchmark -k "gpu" --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
|
|
gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_gpu_${SHORT_SHA}.csv
|
|
|
|
- name: Validate Vulkan Models
|
|
if: matrix.suite == 'vulkan'
|
|
run: |
|
|
cd $GITHUB_WORKSPACE
|
|
PYTHON=python${{ matrix.python-version }} ./setup_venv.sh
|
|
source shark.venv/bin/activate
|
|
pytest -k 'vulkan' --ignore=shark/tests/test_shark_importer.py --ignore=benchmarks/tests/test_hf_benchmark.py --ignore=benchmarks/tests/test_benchmark.py
|