test-models.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Validate Models on Shark Runtime

on:
  push:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'shark/examples/**'
  pull_request:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'shark/examples/**'
  workflow_dispatch:

# Ensure that only a single job or workflow using the same
# concurrency group will run at a time. This would cancel
# any in-progress jobs in the same github workflow and github
# ref (e.g. refs/heads/main or refs/pull/<pr_number>/merge).
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-validate:
    strategy:
      fail-fast: true
      matrix:
        os: [7950x, icelake, a100, MacStudio, ubuntu-latest]
        suite: [cpu,cuda,vulkan]
        python-version: ["3.11"]
        include:
          - os: ubuntu-latest
            suite: lint
          - os: MacStudio
            suite: metal
        exclude:
          - os: ubuntu-latest
            suite: vulkan
          - os: ubuntu-latest
            suite: cuda
          - os: ubuntu-latest
            suite: cpu
          - os: MacStudio
            suite: cuda
          - os: MacStudio
            suite: cpu
          - os: MacStudio
            suite: vulkan
          - os: icelake
            suite: vulkan
          - os: icelake
            suite: cuda
          - os: a100
            suite: cpu
          - os: 7950x
            suite: cpu
          - os: 7950x
            suite: cuda

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v3
    
    - name: Set Environment Variables
      if: matrix.os != '7950x'
      run: |
        echo "SHORT_SHA=`git rev-parse --short=4 HEAD`" >> $GITHUB_ENV
        echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
        
    - name: Set up Python Version File ${{ matrix.python-version }}
      if: matrix.os == 'a100' ||  matrix.os == 'ubuntu-latest' ||  matrix.os == 'icelake'
      run: |
        # See https://github.com/actions/setup-python/issues/433
        echo ${{ matrix.python-version }} >> $GITHUB_WORKSPACE/.python-version
    
    - name: Set up Python ${{ matrix.python-version }}
      if: matrix.os == 'a100' ||  matrix.os == 'ubuntu-latest' ||  matrix.os == 'icelake'
      uses: actions/setup-python@v4
      with:
        python-version: '${{ matrix.python-version }}'
        #cache: 'pip'
        #cache-dependency-path: |
        #  **/requirements-importer.txt
        #  **/requirements.txt
          
    - name: Install dependencies
      if: matrix.suite == 'lint'
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8 pytest toml black
        
    - name: Lint with flake8
      if: matrix.suite == 'lint'
      run: |
        # black format check
        black --version
        black --check .
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --isolated --count --exit-zero --max-complexity=10 --max-line-length=127 \
          --statistics --exclude lit.cfg.py

    - name: Validate Models on CPU
      if: matrix.suite == 'cpu'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} IMPORTER=1 ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --forked --benchmark=native --ci --ci_sha=${SHORT_SHA} --update_tank --tank_url="gs://shark_tank/nightly/" -k cpu 
        gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv
        gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cpu_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cpu_latest.csv
        python build_tools/vicuna_testing.py

    - name: Validate Models on NVIDIA GPU
      if: matrix.suite == 'cuda'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --forked --benchmark=native --ci --ci_sha=${SHORT_SHA} --update_tank --tank_url="gs://shark_tank/nightly/" -k cuda
        gsutil cp ./bench_results.csv gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv
        gsutil cp gs://shark-public/builder/bench_results/${DATE}/bench_results_cuda_${SHORT_SHA}.csv gs://shark-public/builder/bench_results/latest/bench_results_cuda_latest.csv
        # Disabled due to black image bug
        # python build_tools/stable_diffusion_testing.py --device=cuda 

    - name: Validate Vulkan Models (MacOS)
      if: matrix.suite == 'metal' && matrix.os == 'MacStudio'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} ./setup_venv.sh
        source shark.venv/bin/activate
        echo $PATH
        pip list | grep -E "torch|iree"
      # disabled due to a low-visibility memory issue with pytest on macos.
      # pytest --ci --ci_sha=${SHORT_SHA} --local_tank_cache="/Volumes/builder/anush/shark_cache" --tank_url="gs://shark_tank/nightly/" -k metal

    - name: Validate Vulkan Models (a100)
      if: matrix.suite == 'vulkan' && matrix.os == 'a100'
      run: |
        cd $GITHUB_WORKSPACE
        PYTHON=python${{ matrix.python-version }} ./setup_venv.sh
        source shark.venv/bin/activate
        pytest --forked --benchmark="native" --ci --ci_sha=${SHORT_SHA} --update_tank --tank_url="gs://shark_tank/nightly/" -k vulkan
        python build_tools/stable_diffusion_testing.py --device=vulkan

    - name: Validate Vulkan Models (Windows)
      if: matrix.suite == 'vulkan' && matrix.os == '7950x'
      run: |
        ./setup_venv.ps1
        pytest -k vulkan -s --ci

    - name: Validate Stable Diffusion Models (Windows)
      if: matrix.suite == 'vulkan' && matrix.os == '7950x'
      run: |
        ./setup_venv.ps1
        python process_skipfiles.py
        pyinstaller .\apps\stable_diffusion\shark_sd.spec
        python build_tools/stable_diffusion_testing.py --device=vulkan