Merge branch 'main' of github.com:ESMValGroup/ESMValCore into experim…

…ents_first

.circleci/config.yml

@@ -2,8 +2,8 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@13.13.7
-  codecov: codecov/codecov@3.2.5
+  coverage-reporter: codacy/coverage-reporter@14.0.0
+  codecov: codecov/codecov@4.1.0
 
 commands:
   check_changes:
@@ -48,8 +48,15 @@ commands:
   test_installation_from_source:
     parameters:
       extra:
+        description: pip "extra"s to install
         type: string
+        default: "test"
       flags:
+        description: pip install flags
+        type: string
+        default: ""
+      upstream_packages:
+        description: List of packages that will be installed with pip.
         type: string
         default: ""
     steps:
@@ -74,20 +81,25 @@ commands:
               - r-yaml
               - ncl
             " >> environment.yml
+            # Installation of development version of packages requires compilers
+            if [[ "<< parameters.upstream_packages >>" ]]; then
+                echo "  - compilers" >> environment.yml
+            fi
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            mamba env create |& tee /logs/conda.txt
+            git stash  # Restore repository state to get clean version number.
             conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            pip install << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
       - run:
           name: Log versions
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -116,7 +128,7 @@ jobs:
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
             conda activate esmvaltool
-            pip install .[test] > /logs/install.txt 2>&1
+            pip install .[test] |& tee -a /logs/install.txt
       - test_and_report
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -141,8 +153,7 @@ jobs:
       - image: condaforge/mambaforge
     resource_class: large
     steps:
-      - test_installation_from_source:
-          extra: test
+      - test_installation_from_source
 
   test_installation_from_source_develop_mode:
     # Test development installation
@@ -154,6 +165,25 @@ jobs:
           extra: develop
           flags: "--editable"
 
+  test_with_upstream_developments:
+    # Test with development versions of upstream packages
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          upstream_packages: >-
+            git+https://github.com/esgf/esgf-pyclient
+            git+https://github.com/euro-cordex/py-cordex
+            git+https://github.com/SciTools/cartopy
+            git+https://github.com/SciTools/cf-units
+            git+https://github.com/SciTools/iris
+            git+https://github.com/SciTools/iris-grib
+            git+https://github.com/SciTools/nc-time-axis
+            git+https://github.com/SciTools-incubator/iris-esmf-regrid
+            git+https://github.com/SciTools-incubator/python-stratify
+            git+https://github.com/Toblerity/Fiona
+
   test_installation_from_conda:
     # Test conda package installation
     working_directory: /esmvaltool
@@ -167,14 +197,13 @@ jobs:
             set -x
             # Install prerequisites
             mkdir /logs
-            # conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
             mamba create -y --name esmvaltool 'python=3.11'
             set +x; conda activate esmvaltool; set -x
             # Install
             mamba install -y esmvalcore
             # Log versions
-            conda env export > /logs/environment.yml
+            conda env export | tee /logs/environment.yml
             # Test installation
             esmvaltool version
 
@@ -194,9 +223,9 @@ jobs:
             conda activate esmvaltool
             pip install .[doc]
             # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
             # Test building documentation
             MPLBACKEND=Agg sphinx-build -W doc doc/build
       - store_artifacts:
@@ -206,8 +235,9 @@ workflows:
   commit:
     jobs:
       - run_tests
-      - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
 
   nightly:
     triggers:
@@ -218,8 +248,9 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
       - build_documentation
-      - test_installation_from_source_develop_mode
+      - run_tests
       - test_installation_from_conda
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments

.github/workflows/build-and-deploy-on-pypi.yml

@@ -19,13 +19,13 @@ jobs:
       # IMPORTANT: this permission is mandatory for trusted publishing
       id-token: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install pep517
         run: >-
           python -m

.github/workflows/citation_file_validator.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/cffconvert-github-action@2.0.0
         with:

.github/workflows/create-condalock-file.yml

@@ -20,14 +20,14 @@ jobs:
     name: Create and verify conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.11"
+          python-version: "3.12"
           miniforge-version: "latest"
           miniforge-variant: Mambaforge
           use-mamba: true
@@ -38,7 +38,7 @@ jobs:
           conda list
           conda config --show-sources
           conda config --show
-          # setup-miniconda@v2 installs an old conda and mamba
+          # setup-miniconda@v3 installs an old conda and mamba
           # forcing a modern mamba updates both mamba and conda
           conda install -c conda-forge "mamba>=1.4.8"
           conda --version

.github/workflows/install-from-conda.yml

@@ -39,13 +39,13 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
@@ -66,7 +66,7 @@ jobs:
           esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_Linux_python_${{ matrix.python-version }}
           path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -75,12 +75,14 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
+          architecture: ${{ matrix.architecture }}
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
           miniforge-variant: Mambaforge
@@ -100,7 +102,7 @@ jobs:
           esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_OSX_python_${{ matrix.python-version }}
           path: conda_install_osx_artifacts_python_${{ matrix.python-version }}

.github/workflows/install-from-condalock-file.yml

@@ -29,14 +29,14 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool-fromlock
           python-version: ${{ matrix.python-version }}
@@ -55,7 +55,7 @@ jobs:
       - run: pytest -n 2 -m "not installation"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Source_Install_Linux_python_${{ matrix.python-version }}
           path: source_install_linux_artifacts_python_${{ matrix.python-version }}

.github/workflows/install-from-pypi.yml

@@ -39,14 +39,14 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvalcore
           environment-file: environment.yml
@@ -68,7 +68,7 @@ jobs:
           esmvaltool version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: PIP_Install_Linux_python_${{ matrix.python-version }}
           path: pip_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -77,13 +77,15 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
         with:
+          architecture: ${{ matrix.architecture }}
           activate-environment: esmvalcore
           environment-file: environment.yml
           python-version: ${{ matrix.python-version }}
@@ -104,7 +106,7 @@ jobs:
           esmvaltool version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: PIP_Install_OSX_python_${{ matrix.python-version }}
           path: pip_install_osx_artifacts_python_${{ matrix.python-version }}