AMD-SHARK-Studio/.github/workflows/test-models.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Validate Models on Shark Runtime

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:

# Ensure that only a single job or workflow using the same
# concurrency group will run at a time. This would cancel
# any in-progress jobs in the same github workflow and github
# ref (e.g. refs/heads/main or refs/pull/<pr_number>/merge).
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-validate:
    strategy:
      fail-fast: true
      matrix:
        os: [icelake, a100, MacStudio, ubuntu-latest]
        suite: [cpu,cuda,vulkan]
        python-version: ["3.10"]
        include:
          - os: ubuntu-latest
            suite: lint
        exclude:
          - os: ubuntu-latest
            suite: vulkan
          - os: ubuntu-latest
            suite: cuda
          - os: ubuntu-latest
            suite: cpu
          - os: MacStudio
            suite: vulkan
          - os: MacStudio
            suite: cuda
          - os: MacStudio
            suite: cpu
          - os: icelake
            suite: vulkan
          - os: icelake
            suite: cuda
          - os: a100
            suite: cpu

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v3
    
    - name: Set Environment Variables
      run: |
        echo "SHORT_SHA=`git rev-parse --short=4 HEAD`" >> $GITHUB_ENV
        echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
        
    - name: Set up Python Version File ${{ matrix.python-version }}
      if: matrix.os == 'a100' ||  matrix.os == 'ubuntu-latest' ||  matrix.os == 'icelake'
      run: |
        # See https://github.com/actions/setup-python/issues/433
        echo ${{ matrix.python-version }} >> $GITHUB_WORKSPACE/.python-version
    
    - name: Set up Python ${{ matrix.python-version }}
      if: matrix.os == 'a100' ||  matrix.os == 'ubuntu-latest' ||  matrix.os == 'icelake'
      uses: actions/setup-python@v4
      with:
        python-version: '${{ matrix.python-version }}'
        #cache: 'pip'
        #cache-dependency-path: |
        #  **/requirements-importer.txt
        #  **/requirements.txt
          
    - name: Install dependencies
      if: matrix.suite == 'lint'
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8 pytest toml black
        
    - name: Lint with flake8
      if: matrix.suite == 'lint'
      run: |
        # black format check
        black --version
        black --line-length 79 --check .
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude lit.cfg.py
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude lit.cfg.py

    - name: Validate Models on CPU
      if: matrix.suite == 'cpu'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} BENCHMARK=1 IMPORTER=1 ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --benchmark --ci --ci_sha=${SHORT_SHA} -s --local_tank_cache="/data/anush/shark_cache" tank/test_models.py -k cpu --update_tank
        gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv
        gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cpu_latest.csv

    - name: Validate Models on NVIDIA GPU
      if: matrix.suite == 'cuda'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} BENCHMARK=1 IMPORTER=1 ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --benchmark --ci --ci_sha=${SHORT_SHA} -s --local_tank_cache="/data/anush/shark_cache" tank/test_models.py -k cuda --update_tank
        gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv
        gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cuda_latest.csv

    - name: Validate Vulkan Models (MacOS)
      if: matrix.suite == 'vulkan' && matrix.os == 'MacStudio'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
        source shark.venv/bin/activate
        echo "VULKAN SDK PATH wo setup: $VULKAN_SDK"
        cd /Users/anush/VulkanSDK/1.3.224.1/
        source setup-env.sh
        cd $GITHUB_WORKSPACE
        echo "VULKAN SDK PATH with setup: $VULKAN_SDK"
        echo $PATH
        pip list | grep -E "torch|iree"
        pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" tank/test_models.py -k vulkan --update_tank

    - name: Validate Vulkan Models (a100)
      if: matrix.suite == 'vulkan' && matrix.os != 'MacStudio'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} BENCHMARK=1 IMPORTER=1 ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --ci --ci_sha=${SHORT_SHA} -s --local_tank_cache="/data/anush/shark_cache" tank/test_models.py -k vulkan --update_tank