test.yml

name: Test
on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'src/helm-frontend/**'
  pull_request:
    paths-ignore:
      - 'src/helm-frontend/**'

jobs:
  install:
    # Tests that the HELM CLI commands work when only installing required dependencies
    # without optional extra dependencies.
    name: Install
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }}
          restore-keys: |
            pip-
      - run: pip install -e .
      - run: helm-run -h
      - run: helm-summarize -h
      - run: echo "Finished installation."

  test:
    name: Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-
      # Installs dependencies and performs static code checks
      - run: python3 -m pip install virtualenv && python3 -m virtualenv -p python3 venv
      - run: source venv/bin/activate && ./install-dev.sh
      - run: source venv/bin/activate && ./pre-commit.sh
      - name: Run tests
        run: source venv/bin/activate && pytest
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'main' }}
      - name: Run entire pipeline quickly without any data
        # Checking RunSpecs with openai/davinci should be comprehensive enough
        run: source venv/bin/activate && helm-run --suite test -m 100 --skip-instances --models-to-run openai/davinci --exit-on-error