Merge pull request #1183 from lkstrp/pinned-env

ci: adds fixed env ci
pypsa-meets-earth · Nov 21, 2024 · 0eac2ea · 0eac2ea
2 parents 418f0b9 + d4e3f8a
commit 0eac2ea
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 586 deletions.
diff --git a/.github/workflows/ci.yml → .github/workflows/test.yml b/.github/workflows/ci.yml → .github/workflows/test.yml
@@ -1,4 +1,4 @@
-name: CI
+name: Test workflows
 
 on:
   push:
@@ -10,21 +10,29 @@ on:
   schedule:
   - cron: "0 5 * * TUE"
 
+# Cancel any in-progress runs when a new run is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   CACHE_NUMBER: 1  # Change this value to manually reset the environment cache
 
 jobs:
-  build:
+  run-tests:
+    name: OS
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
-      max-parallel: 3
       matrix:
-        os:
-        - ubuntu-latest
-        - macos-latest
-        # - windows-latest
-
-    runs-on: ${{ matrix.os }}
+        os: [ubuntu, macos, windows]
+        include:
+        - os: ubuntu
+          env_file: envs/linux-pinned.yaml
+        - os: macos
+          env_file: envs/macos-pinned.yaml
+        # - os: windows
+        #   env_file: envs/windows-pinned.yaml
 
     defaults:
       run:
@@ -38,7 +46,7 @@ jobs:
       uses: mamba-org/setup-micromamba@v2
       with:
         micromamba-version: '1.5.9-1'
-        environment-file: envs/environment.yaml
+        environment-file: ${{ matrix.env_file }}
         log-level: debug
         init-shell: bash
         cache-environment: true
@@ -64,6 +72,17 @@ jobs:
     - name: Run Test
       run: make test
 
+    - name: Upload artifacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.os }}
+        path: |
+          logs
+          .snakemake/log
+          results
+        retention-days: 3
+
       # - name: Test plotting and summaries
       #   run: |
       #     snakemake --cores all plot_all_p_nom

diff --git a/.github/workflows/update-pinned-env.yml b/.github/workflows/update-pinned-env.yml
@@ -0,0 +1,85 @@
+name: Update pinned envs
+
+on:
+  push:
+    paths:
+    - envs/environment.yaml
+  # Run every Sunday at 5:00 UTC
+  schedule:
+  - cron: "0 5 * * 0"
+  workflow_dispatch:
+
+
+jobs:
+  update-pinned-environment:
+    if: ${{ github.ref == 'refs/heads/main' }}
+    name: Update pinned envs
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        include:
+        - os: ubuntu
+          suffix: "linux"
+        - os: macos
+          suffix: "macos"
+        - os: windows
+          suffix: "windows"
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: ${{ github.event.repository.name }}
+        environment-file: envs/environment.yaml
+
+    - name: Update pinned environment per OS
+      run: |
+        conda env export --name ${{ github.event.repository.name }} --no-builds > envs/${{ matrix.suffix }}-pinned.yaml
+    
+    - name: Add SPDX header
+      if: ${{ matrix.suffix != 'windows' }}
+      run: |
+        SPDX_HEADER="# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors\n# SPDX-License-Identifier: CC0-1.0\n"
+        echo -e "$SPDX_HEADER" | cat - envs/${{ matrix.suffix }}-pinned.yaml > temp && mv temp envs/${{ matrix.suffix }}-pinned.yaml
+
+    - name: Add SPDX header (windows)
+      if: ${{ matrix.suffix == 'windows' }}
+      run: |
+        $SPDX_HEADER = "# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur`r`n# SPDX-License-Identifier: CC0-1.0`r`n`r`n"
+        $CurrentContent = Get-Content "envs/${{ matrix.suffix }}-pinned.yaml" -Raw
+        $NewContent = $SPDX_HEADER + $CurrentContent
+        $NewContent | Set-Content "envs/${{ matrix.suffix }}-pinned.yaml"
+
+    - name: Cache environment files
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.suffix }}-pinned
+        path: envs/${{ matrix.suffix }}-pinned.yaml
+
+  create-pull-request:
+    needs: update-pinned-environment
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download all artifacts
+      uses: actions/download-artifact@v4
+
+    - name: Prepare files for commit
+      run: |
+        mkdir -p envs
+        mv linux-pinned/* envs/linux-pinned.yaml
+        mv macos-pinned/* envs/macos-pinned.yaml
+        mv windows-pinned/* envs/windows-pinned.yaml
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v7
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: update-pinned-environment
+        title: "[github-actions.ci] Update pinned envs"
+        body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux."
+        commit-message: "Update pinned environment files for all platforms"
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
@@ -51,6 +51,12 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o
 
 * Drop entries that contain non-string elements in country column of `CO2_emissions_csv` data in `prepare_transport_data_input.py` script `PR #1166 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1166>`_
 
+* Adds `Dependabot <https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide>`__ to keep GitHub actions up to date. `PR #1184 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1184>`__ 
+
+* Adds code security scans via `CodeQL <https://codeql.github.com/>`__ to CI. `PR #1185 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1185>`__ 
+
+* Adds CI to update keep pinned environment files up to date. `PR #1183 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1183>`__
+
 PyPSA-Earth 0.4.1
 =================