Merge remote-tracking branch 'origin/main' into nested_config

ESMValGroup · Jul 22, 2024 · cf3ad29 · cf3ad29
2 parents 81b6a68 + 546937f
commit cf3ad29
Show file tree

Hide file tree

Showing 31 changed files with 986 additions and 173 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,8 +2,8 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@13.16.5
-  codecov: codecov/codecov@3.2.5
+  coverage-reporter: codacy/coverage-reporter@14.0.0
+  codecov: codecov/codecov@4.1.0
 
 commands:
   check_changes:
@@ -48,8 +48,15 @@ commands:
   test_installation_from_source:
     parameters:
       extra:
+        description: pip "extra"s to install
         type: string
+        default: "test"
       flags:
+        description: pip install flags
+        type: string
+        default: ""
+      upstream_packages:
+        description: List of packages that will be installed with pip.
         type: string
         default: ""
     steps:
@@ -74,20 +81,25 @@ commands:
               - r-yaml
               - ncl
             " >> environment.yml
+            # Installation of development version of packages requires compilers
+            if [[ "<< parameters.upstream_packages >>" ]]; then
+                echo "  - compilers" >> environment.yml
+            fi
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            mamba env create |& tee /logs/conda.txt
+            git stash  # Restore repository state to get clean version number.
             conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            pip install << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
       - run:
           name: Log versions
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -116,7 +128,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             conda activate esmvaltool
-            pip install .[test] > /logs/install.txt 2>&1
+            pip install .[test] |& tee -a /logs/install.txt
       - test_and_report
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -141,8 +153,7 @@ jobs:
       - image: condaforge/mambaforge
     resource_class: large
     steps:
-      - test_installation_from_source:
-          extra: test
+      - test_installation_from_source
 
   test_installation_from_source_develop_mode:
     # Test development installation
@@ -154,6 +165,25 @@ jobs:
           extra: develop
           flags: "--editable"
 
+  test_with_upstream_developments:
+    # Test with development versions of upstream packages
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          upstream_packages: >-
+            git+https://github.com/esgf/esgf-pyclient
+            git+https://github.com/euro-cordex/py-cordex
+            git+https://github.com/SciTools/cartopy
+            git+https://github.com/SciTools/cf-units
+            git+https://github.com/SciTools/iris
+            git+https://github.com/SciTools/iris-grib
+            git+https://github.com/SciTools/nc-time-axis
+            git+https://github.com/SciTools-incubator/iris-esmf-regrid
+            git+https://github.com/SciTools-incubator/python-stratify
+            git+https://github.com/Toblerity/Fiona
+
   test_installation_from_conda:
     # Test conda package installation
     working_directory: /esmvaltool
@@ -167,14 +197,13 @@ jobs:
             set -x
             # Install prerequisites
             mkdir /logs
-            # conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
             mamba create -y --name esmvaltool 'python=3.11'
             set +x; conda activate esmvaltool; set -x
             # Install
             mamba install -y esmvalcore
             # Log versions
-            conda env export > /logs/environment.yml
+            conda env export | tee /logs/environment.yml
             # Test installation
             esmvaltool version
 
@@ -194,9 +223,9 @@ jobs:
             conda activate esmvaltool
             pip install .[doc]
             # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
             # Test building documentation
             MPLBACKEND=Agg sphinx-build -W doc doc/build
       - store_artifacts:
@@ -206,8 +235,9 @@ workflows:
   commit:
     jobs:
       - run_tests
-      - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
 
   nightly:
     triggers:
@@ -218,8 +248,9 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
       - build_documentation
-      - test_installation_from_source_develop_mode
+      - run_tests
       - test_installation_from_conda
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml
@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
@@ -75,7 +75,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -29,7 +29,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:

diff --git a/.github/workflows/install-from-pypi.yml b/.github/workflows/install-from-pypi.yml
@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
@@ -77,7 +77,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml
@@ -37,7 +37,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -74,7 +74,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml
@@ -22,7 +22,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -55,7 +55,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -39,7 +39,7 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
@@ -73,7 +73,7 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}

diff --git a/.zenodo.json b/.zenodo.json
@@ -204,6 +204,11 @@
         {
             "affiliation": "DLR, Germany",
             "name": "Cammarano, Diego"
+        },
+        {
+            "affiliation": "ACCESS-NRI, Australia",
+            "name": "Yousong, Zeng",
+            "orcid": "0000-0002-8385-5367"
         }
     ],
     "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.",

diff --git a/CITATION.cff b/CITATION.cff
@@ -208,13 +208,18 @@ authors:
     affiliation: "DLR, Germany"
     family-names: Cammarano
     given-names: Diego
+  - 
+    affiliation: "ACCESS-NRI, Australia"
+    family-names: Yousong
+    given-names: Zeng
+    orcid: "https://orcid.org/0000-0002-8385-5367"
 
 cff-version: 1.2.0
-date-released: 2024-06-20
+date-released: 2024-07-03
 doi: "10.5281/zenodo.3387139"
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ESMValGroup/ESMValCore/"
 title: ESMValCore
-version: "v2.11.0rc2"
+version: "v2.11.0"
 ...