Major CI cleanup (#293)

* Major CI cleanup

* .

* .

* .

* updates from other pr

* .

* add aarch64, ppc64le

* .

* fix no-test

* add 3.11 tests for windows and macos

* disable doctest for windows

* don't build for aarch64 in PRs

* .

* fix bad merge conflict

---------

Co-authored-by: Marc-Antoine Schmidt <marc-antoine.schmidt@quantco.com>

.github/dependabot.yml

@@ -1,8 +1,8 @@
 version: 2
 updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
+  - package-ecosystem: github-actions
+    directory: /
     schedule:
-      interval: "weekly"
+      interval: weekly
     reviewers:
-      - "quantco/ci"
+      - quantco/ci

.github/workflows/build_wheels_release.yml → .github/workflows/build-wheels.yml

@@ -1,6 +1,7 @@
 name: Build and upload to PyPI
 
 on:
+  pull_request:
   release:
     types:
       - published
@@ -20,15 +21,25 @@ jobs:
         uses: docker/setup-qemu-action@v2
         with:
           platforms: all
+
       - name: Set up micromamba
         uses: mamba-org/setup-micromamba@d05808540d968a55ca33c798e0661fb98f533c73
+
       - name: Build wheels
+        if: github.event_name != 'release'
         uses: pypa/cibuildwheel@v2.15.0
         env:
-          CIBW_ARCHS_LINUX: auto aarch64
+          CIBW_ARCHS_LINUX: auto
+          CIBW_ARCHS_MACOS: x86_64 arm64
+          CIBW_TEST_SKIP: '*-macosx_arm64'
+
+      - name: Build wheels (release)
+        if: github.event_name == 'release' && github.event.action == 'published'
+        uses: pypa/cibuildwheel@v2.15.0
+        env:
+          CIBW_ARCHS_LINUX: aarch64 aarch64
           CIBW_ARCHS_MACOS: x86_64 arm64
           CIBW_TEST_SKIP: '*-macosx_arm64'
-
       - uses: actions/upload-artifact@v3
         with:
           path: ./wheelhouse/*.whl
@@ -55,6 +66,7 @@ jobs:
           path: dist/*.tar.gz
 
   upload_testpypi:
+    if: github.event_name == 'release' && github.event.action == 'published'
     needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
     steps:
@@ -67,9 +79,10 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.GH_TESTPYPI_UPLOAD }}
-          repository_url: https://test.pypi.org/legacy/
+          repository-url: https://test.pypi.org/legacy/
 
   upload_pypi:
+    if: github.event_name == 'release' && github.event.action == 'published'
     needs: [build_wheels, build_sdist, upload_testpypi]
     runs-on: ubuntu-latest
     steps:

.github/workflows/ci.yml

@@ -2,64 +2,77 @@ name: CI
 on: [push]
 
 jobs:
-  linter:
-    name: "Linter (pre-commit)"
+  pre-commit-checks:
+    name: pre-commit checks
+    timeout-minutes: 30
     runs-on: ubuntu-latest
     env:
-      CI: True
-    strategy:
-      fail-fast: true
-    steps:
-     - name: Pull image
-       run: docker pull condaforge/mambaforge:latest
-     - name: Checkout branch
-       uses: actions/checkout@v3
-       with:
-         ref: ${{ github.head_ref }}
-     - name: Run linter inside of container
-       uses: ./.github/actions/linter
-  linux:
-    name: "Linux - tests - Py${{ matrix.PYTHON_VERSION }}"
-    runs-on: ubuntu-latest
-    env:
-      CI: True
-    strategy:
-      fail-fast: true
-      matrix:
-        PYTHON_VERSION: ['3.7', '3.8', '3.9', '3.10']
+      PRE_COMMIT_USE_MICROMAMBA: 1
     steps:
-     - name: Pull image
-       run: docker pull condaforge/mambaforge:latest
-     - name: Checkout branch
-       uses: actions/checkout@v3
-       with:
-         ref: ${{ github.head_ref }}
-     - name: Run CI inside of container
-       uses: ./.github/actions/unittests
-       with:
-         python_version: ${{ matrix.PYTHON_VERSION }}
-  linux-conda-build:
-    name: "Linux - conda-build - Python ${{ matrix.CONDA_BUILD_YML }}"
-    runs-on: ubuntu-latest
-    env:
-      CI: True
+      - name: Checkout branch
+        uses: actions/checkout@v3
+      - name: Set up micromamba
+        uses: mamba-org/setup-micromamba@d05808540d968a55ca33c798e0661fb98f533c73
+      - name: Add micromamba to GITHUB_PATH
+        run: echo "${HOME}/micromamba-bin" >> "$GITHUB_PATH"
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Run pre-commit checks
+        uses: pre-commit/action@v3.0.0
+
+  unit-tests:
+    name: Tests - ${{ matrix.os }} - Py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
-        CONDA_BUILD_YML:
-          - linux_64_python3.7_default.____cpython
-          - linux_aarch64_python3.10_default.____cpython
-          - linux_ppc64le_python3.10_default.____cpython
+        include:
+          - { os: ubuntu-latest, python-version: '3.7' }
+          - { os: ubuntu-latest, python-version: '3.8' }
+          - { os: ubuntu-latest, python-version: '3.9' }
+          - { os: ubuntu-latest, python-version: '3.10' }
+          - { os: ubuntu-latest, python-version: '3.11' }
+          - { os: macos-latest, python-version: '3.7' }
+          - { os: macos-latest, python-version: '3.11' }
+          - { os: windows-latest, python-version: '3.7' }
+          - { os: windows-latest, python-version: '3.11' }
     steps:
-      - name: Pull image
-        run: docker pull condaforge/mambaforge:latest
       - name: Checkout branch
         uses: actions/checkout@v3
+      - name: Set up conda env
+        uses: mamba-org/setup-micromamba@875557da4ee020f18df03b8910a42203fbf02da1
         with:
-          ref: ${{ github.head_ref }}
-      - name: Fetch full git history
-        run: git fetch --prune --unshallow
-      - name: Run CI inside of container
-        uses: ./.github/actions/conda-build
-        with:
-          conda_build_yml: ${{ matrix.CONDA_BUILD_YML }}
+          environment-file: ${{ matrix.os == 'windows-latest' && 'environment-win.yml' || 'environment.yml' }}
+          init-shell: ${{ matrix.os == 'windows-latest' && 'powershell' || 'bash' }}
+          cache-environment: true
+          create-args: >-
+            python=${{ matrix.python-version }}
+      - name: Install repository (unix)
+        if: matrix.os != 'windows-latest'
+        shell: bash -el {0}
+        run: pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
+      - name: Install repository (windows)
+        if: matrix.os == 'windows-latest'
+        shell: powershell
+        run: pip install --no-use-pep517 --no-deps --disable-pip-version-check -e .
+      - name: Run pytest (unix)
+        if: matrix.os != 'windows-latest'
+        shell: bash -el {0}
+        run: pytest -nauto tests -m "not high_memory" --doctest-modules src/
+      - name: Run pytest (windows)
+        if: matrix.os == 'windows-latest'
+        shell: powershell
+        run: pytest -nauto tests -m "not high_memory"
+      - name: Run doctest
+        if: matrix.os != 'windows-latest'
+        shell: bash -el {0}
+        # Check that the readme example will work by running via doctest.
+        # We run outside the repo to make the test a bit more similar to
+        # a user running after installing with conda.
+        run: |
+          mkdir ../temp
+          cp README.md ../temp
+          cd ../temp
+          python -m doctest -v README.md