Add ComputeNode, LocalV2 + deprecate RemoteV1 and LocalV1
#611
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python package | |
| on: [push, pull_request] | |
| permissions: | |
| # Minimum permissions required by skip-duplicate-actions | |
| actions: write | |
| contents: read | |
| jobs: | |
| linting: | |
| name: Run linting/pre-commit checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install pre-commit | |
| - run: pre-commit --version | |
| - run: pre-commit install | |
| - run: pre-commit run --all-files | |
| unit-tests: | |
| needs: [linting] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| platform: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| env: | |
| PLATFORM: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # When pushing to the same branch, cancel the previous workflow if it is still running. | |
| - uses: fkirc/skip-duplicate-actions@v5.3.1 | |
| with: | |
| cancel_others: true | |
| - name: Install poetry | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: poetry | |
| - name: Install dependencies | |
| run: poetry install --with=dev | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| if: runner.os == 'Linux' | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Test with pytest | |
| run: poetry run pytest --cov=milatools --cov-report=xml --cov-append | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: unittests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| mock-slurm-integration-tests: | |
| name: integration tests with a mock slurm cluster | |
| needs: [unit-tests] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| platform: [ubuntu-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| # For the action to work, you have to supply a mysql | |
| # service as defined below. | |
| services: | |
| mysql: | |
| image: mysql:8.0 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| ports: | |
| - "8888:3306" | |
| options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # When pushing to the same branch, cancel the previous workflow if it is still running. | |
| - uses: fkirc/skip-duplicate-actions@v5.3.1 | |
| with: | |
| cancel_others: true | |
| # NOTE: Replacing this with our customized version of | |
| # - uses: koesterlab/setup-slurm-action@v1 | |
| - uses: ./.github/custom_setup_slurm_action | |
| timeout-minutes: 5 | |
| - name: Test if the slurm cluster is setup correctly | |
| run: srun --nodes=1 --ntasks=1 --cpus-per-task=1 --mem=1G --time=00:01:00 hostname | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| timeout-minutes: 5 | |
| env: | |
| SLURM_CLUSTER: localhost | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| real-slurm-integration-tests: | |
| name: integration tests with a real SLURM cluster | |
| needs: [mock-slurm-integration-tests] | |
| runs-on: self-hosted | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| python-version: ['3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # When pushing to the same branch, cancel the previous workflow if it is still running. | |
| - uses: fkirc/skip-duplicate-actions@v5.3.1 | |
| with: | |
| cancel_others: true | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| id: self_hosted_integration_tests | |
| run: poetry run pytest --slow -n 5 --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| # todo: Seems like the tests time out while waiting for jobs to be allocated! | |
| timeout-minutes: 30 | |
| env: | |
| SLURM_CLUSTER: mila | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false |