Transition mila init to RemoteV2
#648
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python package | |
| on: [push, pull_request] | |
| # https://stackoverflow.com/a/72408109/6388696 | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| linting: | |
| name: Run linting/pre-commit checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install pre-commit | |
| - run: pre-commit --version | |
| - run: pre-commit install | |
| - run: pre-commit run --all-files | |
| unit-tests: | |
| needs: [linting] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| platform: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| env: | |
| PLATFORM: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: poetry | |
| - name: Install dependencies | |
| run: poetry install --with=dev | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| if: runner.os == 'Linux' | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Test with pytest | |
| run: poetry run pytest --cov=milatools --cov-report=xml --cov-append | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: unittests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| mock-slurm-integration-tests: | |
| name: integration tests with a mock slurm cluster | |
| needs: [unit-tests] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| platform: [ubuntu-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| # For the action to work, you have to supply a mysql | |
| # service as defined below. | |
| services: | |
| mysql: | |
| image: mysql:8.0 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| ports: | |
| - "8888:3306" | |
| options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # NOTE: Replacing this with our customized version of | |
| # - uses: koesterlab/setup-slurm-action@v1 | |
| - uses: ./.github/custom_setup_slurm_action | |
| timeout-minutes: 5 | |
| - name: Test if the slurm cluster is setup correctly | |
| run: srun --nodes=1 --ntasks=1 --cpus-per-task=1 --mem=1G --time=00:01:00 hostname | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| timeout-minutes: 15 | |
| env: | |
| SLURM_CLUSTER: localhost | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| real-slurm-integration-tests: | |
| name: integration tests with a real SLURM cluster | |
| needs: [mock-slurm-integration-tests] | |
| runs-on: self-hosted | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| python-version: ['3.11'] | |
| # TODO: also run these integration tests on the DRAC clusters (probably less | |
| # frequently because of the long job wait times) | |
| cluster: ['mila'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| id: self_hosted_integration_tests | |
| run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| timeout-minutes: 30 | |
| env: | |
| SLURM_CLUSTER: ${{ matrix.cluster }} | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false |