Merge branch 'main' into endogenous_export

pypsa-meets-earth · Nov 28, 2024 · a347e25 · a347e25
2 parents 20af55a + c9f30e1
commit a347e25
Show file tree

Hide file tree

Showing 12 changed files with 1,383 additions and 256 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,7 +1,6 @@
 on:
-  push:
-    branches:
-    - main
+  schedule:
+  - cron: "0 5 * * 0"
 
 jobs:
   contrib-readme-job:

diff --git a/.github/workflows/ci.yml → .github/workflows/test.yml b/.github/workflows/ci.yml → .github/workflows/test.yml
@@ -1,4 +1,4 @@
-name: CI
+name: Test workflows
 
 on:
   push:
@@ -10,21 +10,29 @@ on:
   schedule:
   - cron: "0 5 * * TUE"
 
+# Cancel any in-progress runs when a new run is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
-  CACHE_NUMBER: 1  # Change this value to manually reset the environment cache
+  CACHE_NUMBER: 0  # Change this value to manually reset the environment cache
 
 jobs:
-  build:
+  run-tests:
+    name: OS
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
-      max-parallel: 3
       matrix:
-        os:
-        - ubuntu-latest
-        - macos-latest
-        # - windows-latest
-
-    runs-on: ${{ matrix.os }}
+        os: [ubuntu, macos] #, windows]
+        include:
+        - os: ubuntu
+          env_file: envs/linux-pinned.yaml
+        - os: macos
+          env_file: envs/macos-pinned.yaml
+        # - os: windows
+        #   env_file: envs/windows-pinned.yaml
 
     defaults:
       run:
@@ -38,7 +46,7 @@ jobs:
       uses: mamba-org/setup-micromamba@v2
       with:
         micromamba-version: '1.5.9-1'
-        environment-file: envs/environment.yaml
+        environment-file: ${{ matrix.env_file }}
         log-level: debug
         init-shell: bash
         cache-environment: true
@@ -64,6 +72,17 @@ jobs:
     - name: Run Test
       run: make test
 
+    - name: Upload artifacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: results-${{ matrix.os }}
+        path: |
+          logs
+          .snakemake/log
+          results
+        retention-days: 3
+
       # - name: Test plotting and summaries
       #   run: |
       #     snakemake --cores all plot_all_p_nom

diff --git a/.github/workflows/update-pinned-env.yml b/.github/workflows/update-pinned-env.yml
@@ -0,0 +1,85 @@
+name: Update pinned envs
+
+on:
+  push:
+    paths:
+    - envs/environment.yaml
+  # Run every Sunday at 5:00 UTC
+  schedule:
+  - cron: "0 5 * * 0"
+  workflow_dispatch:
+
+
+jobs:
+  update-pinned-environment:
+    if: ${{ github.ref == 'refs/heads/main' }}
+    name: Update pinned envs
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu, macos, windows]
+        include:
+        - os: ubuntu
+          suffix: "linux"
+        - os: macos
+          suffix: "macos"
+        - os: windows
+          suffix: "windows"
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: ${{ github.event.repository.name }}
+        environment-file: envs/environment.yaml
+
+    - name: Update pinned environment per OS
+      run: |
+        conda env export --name ${{ github.event.repository.name }} --no-builds > envs/${{ matrix.suffix }}-pinned.yaml
+
+    - name: Add SPDX header
+      if: ${{ matrix.suffix != 'windows' }}
+      run: |
+        SPDX_HEADER="# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur Authors\n#\n# SPDX-License-Identifier: CC0-1.0\n"
+        echo -e "$SPDX_HEADER" | cat - envs/${{ matrix.suffix }}-pinned.yaml > temp && mv temp envs/${{ matrix.suffix }}-pinned.yaml
+
+    - name: Add SPDX header (windows)
+      if: ${{ matrix.suffix == 'windows' }}
+      run: |
+        $SPDX_HEADER = "# SPDX-FileCopyrightText: PyPSA-Earth and PyPSA-Eur`r`n#`r`n# SPDX-License-Identifier: CC0-1.0`r`n`r`n"
+        $CurrentContent = Get-Content "envs/${{ matrix.suffix }}-pinned.yaml" -Raw
+        $NewContent = $SPDX_HEADER + $CurrentContent
+        $NewContent | Set-Content "envs/${{ matrix.suffix }}-pinned.yaml"
+
+    - name: Cache environment files
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.suffix }}-pinned
+        path: envs/${{ matrix.suffix }}-pinned.yaml
+
+  create-pull-request:
+    needs: update-pinned-environment
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Download all artifacts
+      uses: actions/download-artifact@v4
+
+    - name: Prepare files for commit
+      run: |
+        mkdir -p envs
+        mv linux-pinned/* envs/linux-pinned.yaml
+        mv macos-pinned/* envs/macos-pinned.yaml
+        mv windows-pinned/* envs/windows-pinned.yaml
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v7
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: update-pinned-environment
+        title: "[github-actions.ci] Update pinned envs"
+        body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux."
+        commit-message: "Update pinned environment files for all platforms"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -90,6 +90,6 @@ repos:
 
 # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v4.0.3
+  rev: v5.0.2
   hooks:
   - id: reuse
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ by
 
 ## Development Status: **Stable and Active**
 
-[![Test workflows](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml/badge.svg)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml)
+[![Test workflows](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/test.yml/badge.svg)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/test.yml)
 [![Documentation Status](https://readthedocs.org/projects/pypsa-earth/badge/?version=latest)](https://pypsa-earth.readthedocs.io/en/latest/?badge=latest)
 ![Size](https://img.shields.io/github/repo-size/pypsa-meets-earth/pypsa-earth)
 [![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
@@ -393,36 +393,36 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
                     <sub><b>Ekaterina-Vo</b></sub>
                 </a>
             </td>
+            <td align="center">
+                <a href="https://github.com/lkstrp">
+                    <img src="https://avatars.githubusercontent.com/u/62255395?v=4" width="100;" alt="lkstrp"/>
+                    <br />
+                    <sub><b>Lukas Trippe</b></sub>
+                </a>
+            </td>
             <td align="center">
                 <a href="https://github.com/Tooblippe">
                     <img src="https://avatars.githubusercontent.com/u/805313?v=4" width="100;" alt="Tooblippe"/>
                     <br />
                     <sub><b>Tobias</b></sub>
                 </a>
             </td>
+		</tr>
+		<tr>
             <td align="center">
                 <a href="https://github.com/asolavi">
                     <img src="https://avatars.githubusercontent.com/u/131155817?v=4" width="100;" alt="asolavi"/>
                     <br />
                     <sub><b>Null</b></sub>
                 </a>
             </td>
-		</tr>
-		<tr>
             <td align="center">
                 <a href="https://github.com/rajesh-ieg">
                     <img src="https://avatars.githubusercontent.com/u/120284682?v=4" width="100;" alt="rajesh-ieg"/>
                     <br />
                     <sub><b>Null</b></sub>
                 </a>
             </td>
-            <td align="center">
-                <a href="https://github.com/lkstrp">
-                    <img src="https://avatars.githubusercontent.com/u/62255395?v=4" width="100;" alt="lkstrp"/>
-                    <br />
-                    <sub><b>Lukas Trippe</b></sub>
-                </a>
-            </td>
             <td align="center">
                 <a href="https://github.com/danielelerede-oet">
                     <img src="https://avatars.githubusercontent.com/u/175011591?v=4" width="100;" alt="danielelerede-oet"/>

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
@@ -26,12 +26,17 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o
 
 * Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1157>`__
 
+* Removed duplications of devendencies in environment.yaml `PR #1128 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1128>`_
+
 * Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1137>`__
 
+
 * Enable sector rules import in subworkflow `PR #1178 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1178>`__
 
 * Include option of endogenous export, which optimizes the export quantity based on price signals `PR #1201 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1201>`__
 
+* Remove elec-based H2 and battery technologies before addition in `prepare_sector_network.py` script and fix bus names for links that models H2 repuspose network `PR #1198 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1198>`__
+
 **Minor Changes and bug-fixing**
 
 * The default configuration for `electricity:estimate_renewable_capacities:year` was updated from 2020 to 2023. `PR #1106 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1106>`__
@@ -50,6 +55,12 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o
 
 * Drop entries that contain non-string elements in country column of `CO2_emissions_csv` data in `prepare_transport_data_input.py` script `PR #1166 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1166>`_
 
+* Adds `Dependabot <https://docs.github.com/en/code-security/getting-started/dependabot-quickstart-guide>`__ to keep GitHub actions up to date. `PR #1184 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1184>`__
+
+* Adds code security scans via `CodeQL <https://codeql.github.com/>`__ to CI. `PR #1185 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1185>`__
+
+* Adds CI to update keep pinned environment files up to date. `PR #1183 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1183>`__ and  `PR #1210 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/1210>`__
+
 PyPSA-Earth 0.4.1
 =================
 

diff --git a/envs/environment.yaml b/envs/environment.yaml
@@ -13,11 +13,14 @@ dependencies:
 - mamba   # esp for windows build
 
 - pypsa>=0.24, <0.25
-# - atlite>=0.2.4  # until https://github.com/PyPSA/atlite/issues/244 is not merged
 - dask
-- powerplantmatching
-- earth-osm>=2.1
-- atlite
+# currently the packages are being installed with pip
+# need to move back to conda once the issues will be resolved
+- powerplantmatching>=0.5.19
+# - earth-osm>=2.1
+# until the release will incorporate all the fixes needed
+# to work with CDS beta
+- atlite>=0.3
 
   # Dependencies of the workflow itself
 - xlrd
@@ -81,8 +84,7 @@ dependencies:
 - gurobi
 
 - pip:
-  - earth-osm>=2.2 # until conda release it out
-  - powerplantmatching>=0.5.19 # until conda release it out
+  - earth-osm>=2.2 # until conda release it out for earth-osm
   - git+https://github.com/davide-f/google-drive-downloader@master  # google drive with fix for virus scan
   - git+https://github.com/FRESNA/vresutils@master  # until new pip release > 0.3.1 (strictly)
   - tsam>=1.1.0