Update for ORAS5 fix

ESMValGroup · Dec 2, 2024 · e02bdaf · e02bdaf
2 parents 8de7633 + 1cf3862
commit e02bdaf
Show file tree

Hide file tree

Showing 585 changed files with 33,617 additions and 26,113 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,8 +2,8 @@
 version: 2.1
 
 orbs:
-  coverage-reporter: codacy/coverage-reporter@13.16.5
-  codecov: codecov/codecov@3.2.5
+  coverage-reporter: codacy/coverage-reporter@14.0.0
+  codecov: codecov/codecov@4.1.0
 
 commands:
   check_changes:
@@ -21,15 +21,18 @@ commands:
               circleci step halt
             fi
   test_and_report:
+    parameters:
+      args:
+        type: string
+        default: ""
     steps:
       - run:
           name: Run tests
           command: |
             mkdir -p test-reports
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            flake8 -j 4
-            pytest -n 4 --junitxml=test-reports/report.xml
+            pytest -n 4 --junitxml=test-reports/report.xml << parameters.args >>
             esmvaltool version
       - store_test_results:
           path: test-reports/report.xml
@@ -48,8 +51,15 @@ commands:
   test_installation_from_source:
     parameters:
       extra:
+        description: pip "extra"s to install
         type: string
+        default: "test"
       flags:
+        description: pip install flags
+        type: string
+        default: ""
+      upstream_packages:
+        description: List of packages that will be installed with pip.
         type: string
         default: ""
     steps:
@@ -74,20 +84,25 @@ commands:
               - r-yaml
               - ncl
             " >> environment.yml
+            # Installation of development version of packages requires compilers
+            if [[ "<< parameters.upstream_packages >>" ]]; then
+                echo "  - compilers" >> environment.yml
+            fi
             # Install
             . /opt/conda/etc/profile.d/conda.sh
             mkdir /logs
-            mamba env create >> /logs/conda.txt 2>&1
+            mamba env create |& tee /logs/conda.txt
+            git stash  # Restore repository state to get clean version number.
             conda activate esmvaltool
-            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+            pip install << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
       - run:
           name: Log versions
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda activate esmvaltool
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
       - test_and_report
       - save_cache:
           key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -117,7 +132,8 @@ jobs:
             mkdir /logs
             conda activate esmvaltool
             pip install .[test] > /logs/install.txt 2>&1
-      - test_and_report
+      - test_and_report:
+          args: --cov
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
@@ -138,27 +154,45 @@ jobs:
   test_installation_from_source_test_mode:
     # Test installation from source
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/miniforge3
     resource_class: large
     steps:
-      - test_installation_from_source:
-          extra: test
+      - test_installation_from_source
 
   test_installation_from_source_develop_mode:
     # Test development installation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/miniforge3
     resource_class: large
     steps:
       - test_installation_from_source:
           extra: develop
           flags: "--editable"
 
+  test_with_upstream_developments:
+    # Test with development versions of upstream packages
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          upstream_packages: >-
+            git+https://github.com/esgf/esgf-pyclient
+            git+https://github.com/euro-cordex/py-cordex
+            git+https://github.com/SciTools/cartopy
+            git+https://github.com/SciTools/cf-units
+            git+https://github.com/SciTools/iris
+            git+https://github.com/SciTools/iris-grib
+            git+https://github.com/SciTools/nc-time-axis
+            git+https://github.com/SciTools-incubator/iris-esmf-regrid
+            git+https://github.com/SciTools-incubator/python-stratify
+            git+https://github.com/Toblerity/Fiona
+
   test_installation_from_conda:
     # Test conda package installation
     working_directory: /esmvaltool
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/miniforge3
     resource_class: medium
     steps:
       - run:
@@ -167,21 +201,20 @@ jobs:
             set -x
             # Install prerequisites
             mkdir /logs
-            # conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
             mamba create -y --name esmvaltool 'python=3.11'
             set +x; conda activate esmvaltool; set -x
             # Install
             mamba install -y esmvalcore
             # Log versions
-            conda env export > /logs/environment.yml
+            conda env export | tee /logs/environment.yml
             # Test installation
             esmvaltool version
 
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/mambaforge
+      - image: condaforge/miniforge3
     resource_class: medium
     steps:
       - checkout
@@ -194,9 +227,9 @@ jobs:
             conda activate esmvaltool
             pip install .[doc]
             # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
             # Test building documentation
             MPLBACKEND=Agg sphinx-build -W doc doc/build
       - store_artifacts:
@@ -206,8 +239,8 @@ workflows:
   commit:
     jobs:
       - run_tests
-      - test_installation_from_source_test_mode
       - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
 
   nightly:
     triggers:
@@ -218,8 +251,9 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
       - build_documentation
-      - test_installation_from_source_develop_mode
+      - run_tests
       - test_installation_from_conda
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments
diff --git a/.circleci/install_triggers b/.circleci/install_triggers
@@ -1,5 +1,3 @@
 ^\.circleci/
 ^environment\.yml$
 ^pyproject.toml$
-^setup\.py$
-^setup\.cfg$
diff --git a/.editorconfig b/.editorconfig
@@ -27,4 +27,3 @@ indent_size = 2
 
 [*.{md,Rmd}]
 trim_trailing_whitespace = false
-
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -0,0 +1,2 @@
+# Use ruff formatter (#2524)
+436558caacda69d4966a5aff35959ce9188cac37
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1 @@
-esmvalcore/cmor @jvegasbsc
 .github/workflows @valeriupredoi
-
diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml
@@ -19,13 +19,13 @@ jobs:
       # IMPORTANT: this permission is mandatory for trusted publishing
       id-token: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python 3.11
+      - name: Set up Python 3.13
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.13"
       - name: Install pep517
         run: >-
           python -m

diff --git a/.github/workflows/citation_file_validator.yml b/.github/workflows/citation_file_validator.yml
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/cffconvert-github-action@2.0.0
         with:

diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml
@@ -20,16 +20,15 @@ jobs:
     name: Create and verify conda lock file for latest Python
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           activate-environment: esmvaltool-fromlock
-          python-version: "3.11"
+          python-version: "3.13"
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - name: Update and show conda config
         run: |
@@ -38,9 +37,11 @@ jobs:
           conda list
           conda config --show-sources
           conda config --show
-          # setup-miniconda@v2 installs an old conda and mamba
+          # setup-miniconda@v3 installs an old conda and mamba
           # forcing a modern mamba updates both mamba and conda
-          conda install -c conda-forge "mamba>=1.4.8"
+          # unpin mamba after conda-lock=3 release
+          # see github.com/ESMValGroup/ESMValTool/issues/3782
+          conda install -c conda-forge "mamba>=1.4.8,<2"
           conda --version
           mamba --version
       - name: Gather Python info
@@ -71,8 +72,6 @@ jobs:
         run: |
           esmvaltool --help
           esmvaltool version
-      - name: Run flake8
-        run: flake8
       - name: Run pytests
         run: pytest -n 2 -m "not installation"
       # Automated PR

diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml
@@ -39,17 +39,16 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
       # fail-fast set to False allows all other tests
       # in the workflow to run regardless of any fail
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
@@ -66,7 +65,7 @@ jobs:
           esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_Linux_python_${{ matrix.python-version }}
           path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
@@ -75,15 +74,16 @@ jobs:
     runs-on: "macos-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
       fail-fast: false
     name: OSX Python ${{ matrix.python-version }}
     steps:
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
+          architecture: ${{ matrix.architecture }}
           python-version: ${{ matrix.python-version }}
           miniforge-version: "latest"
-          miniforge-variant: Mambaforge
           use-mamba: true
       - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
       - name: Record versions
@@ -100,7 +100,7 @@ jobs:
           esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Conda_Install_OSX_python_${{ matrix.python-version }}
           path: conda_install_osx_artifacts_python_${{ matrix.python-version }}
diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml
@@ -29,14 +29,14 @@ jobs:
     runs-on: "ubuntu-latest"
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
       fail-fast: false
     name: Linux Python ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: esmvaltool-fromlock
           python-version: ${{ matrix.python-version }}
@@ -51,11 +51,10 @@ jobs:
       - run: pip install -e .[develop]
       - run: esmvaltool --help
       - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
-      - run: flake8
       - run: pytest -n 2 -m "not installation"
       - name: Upload artifacts
         if: ${{ always() }}  # upload artifacts even if fail
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: Source_Install_Linux_python_${{ matrix.python-version }}
           path: source_install_linux_artifacts_python_${{ matrix.python-version }}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -27,4 +27,3 @@ indent_size = 2

		[*.{md,Rmd}]
		trim_trailing_whitespace = false
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# Use ruff formatter (#2524)
		436558caacda69d4966a5aff35959ce9188cac37
Original file line number	Diff line number	Diff line change
		@@ -1,3 +1 @@
		esmvalcore/cmor @jvegasbsc
		.github/workflows @valeriupredoi