Skip to content

Commit

Permalink
Merge commit 'f30e19f9761db8262f15667bc3a41fcdb420c925' into icon-xpp
Browse files Browse the repository at this point in the history
  • Loading branch information
schlunma committed Oct 18, 2024
2 parents f0e458a + f30e19f commit 99def59
Show file tree
Hide file tree
Showing 201 changed files with 11,042 additions and 4,499 deletions.
70 changes: 50 additions & 20 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
version: 2.1

orbs:
coverage-reporter: codacy/coverage-reporter@13.13.7
codecov: codecov/codecov@3.2.5
coverage-reporter: codacy/coverage-reporter@14.0.0
codecov: codecov/codecov@4.1.0

commands:
check_changes:
Expand All @@ -28,7 +28,6 @@ commands:
mkdir -p test-reports
. /opt/conda/etc/profile.d/conda.sh
conda activate esmvaltool
flake8 -j 4
pytest -n 4 --junitxml=test-reports/report.xml
esmvaltool version
- store_test_results:
Expand All @@ -48,8 +47,15 @@ commands:
test_installation_from_source:
parameters:
extra:
description: pip "extra"s to install
type: string
default: "test"
flags:
description: pip install flags
type: string
default: ""
upstream_packages:
description: List of packages that will be installed with pip.
type: string
default: ""
steps:
Expand All @@ -74,20 +80,25 @@ commands:
- r-yaml
- ncl
" >> environment.yml
# Installation of development version of packages requires compilers
if [[ "<< parameters.upstream_packages >>" ]]; then
echo " - compilers" >> environment.yml
fi
# Install
. /opt/conda/etc/profile.d/conda.sh
mkdir /logs
mamba env create >> /logs/conda.txt 2>&1
mamba env create |& tee /logs/conda.txt
git stash # Restore repository state to get clean version number.
conda activate esmvaltool
pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
pip install << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
- run:
name: Log versions
command: |
. /opt/conda/etc/profile.d/conda.sh
conda activate esmvaltool
dpkg -l > /logs/versions.txt
conda env export > /logs/environment.yml
pip freeze > /logs/requirements.txt
dpkg -l | tee /logs/versions.txt
conda env export | tee /logs/environment.yml
pip freeze | tee /logs/requirements.txt
- test_and_report
- save_cache:
key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
Expand Down Expand Up @@ -116,7 +127,7 @@ jobs:
. /opt/conda/etc/profile.d/conda.sh
mkdir /logs
conda activate esmvaltool
pip install .[test] > /logs/install.txt 2>&1
pip install .[test] |& tee -a /logs/install.txt
- test_and_report
- save_cache:
key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
Expand All @@ -141,8 +152,7 @@ jobs:
- image: condaforge/mambaforge
resource_class: large
steps:
- test_installation_from_source:
extra: test
- test_installation_from_source

test_installation_from_source_develop_mode:
# Test development installation
Expand All @@ -154,6 +164,25 @@ jobs:
extra: develop
flags: "--editable"

test_with_upstream_developments:
# Test with development versions of upstream packages
docker:
- image: condaforge/mambaforge
resource_class: large
steps:
- test_installation_from_source:
upstream_packages: >-
git+https://github.com/esgf/esgf-pyclient
git+https://github.com/euro-cordex/py-cordex
git+https://github.com/SciTools/cartopy
git+https://github.com/SciTools/cf-units
git+https://github.com/SciTools/iris
git+https://github.com/SciTools/iris-grib
git+https://github.com/SciTools/nc-time-axis
git+https://github.com/SciTools-incubator/iris-esmf-regrid
git+https://github.com/SciTools-incubator/python-stratify
git+https://github.com/Toblerity/Fiona
test_installation_from_conda:
# Test conda package installation
working_directory: /esmvaltool
Expand All @@ -167,14 +196,13 @@ jobs:
set -x
# Install prerequisites
mkdir /logs
# conda update -y conda > /logs/conda.txt 2>&1
# Create and activate conda environment
mamba create -y --name esmvaltool 'python=3.11'
set +x; conda activate esmvaltool; set -x
# Install
mamba install -y esmvalcore
# Log versions
conda env export > /logs/environment.yml
conda env export | tee /logs/environment.yml
# Test installation
esmvaltool version
Expand All @@ -194,9 +222,9 @@ jobs:
conda activate esmvaltool
pip install .[doc]
# Log versions
dpkg -l > /logs/versions.txt
conda env export > /logs/environment.yml
pip freeze > /logs/requirements.txt
dpkg -l | tee /logs/versions.txt
conda env export | tee /logs/environment.yml
pip freeze | tee /logs/requirements.txt
# Test building documentation
MPLBACKEND=Agg sphinx-build -W doc doc/build
- store_artifacts:
Expand All @@ -206,8 +234,9 @@ workflows:
commit:
jobs:
- run_tests
- test_installation_from_source_test_mode
- test_installation_from_source_develop_mode
- test_installation_from_source_test_mode
- test_with_upstream_developments

nightly:
triggers:
Expand All @@ -218,8 +247,9 @@ workflows:
only:
- main
jobs:
- run_tests
- test_installation_from_source_test_mode
- build_documentation
- test_installation_from_source_develop_mode
- run_tests
- test_installation_from_conda
- test_installation_from_source_develop_mode
- test_installation_from_source_test_mode
- test_with_upstream_developments
1 change: 0 additions & 1 deletion .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,3 @@ indent_size = 2

[*.{md,Rmd}]
trim_trailing_whitespace = false

2 changes: 0 additions & 2 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
esmvalcore/cmor @jvegasbsc
.github/workflows @valeriupredoi

6 changes: 3 additions & 3 deletions .github/workflows/build-and-deploy-on-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ jobs:
# IMPORTANT: this permission is mandatory for trusted publishing
id-token: write
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.11
- name: Set up Python 3.12
uses: actions/setup-python@v4
with:
python-version: "3.11"
python-version: "3.12"
- name: Install pep517
run: >-
python -m
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/citation_file_validator.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out a copy of the repository
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Check whether the citation metadata from CITATION.cff is valid
uses: citation-file-format/cffconvert-github-action@2.0.0
with:
Expand Down
10 changes: 4 additions & 6 deletions .github/workflows/create-condalock-file.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ jobs:
name: Create and verify conda lock file for latest Python
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: conda-incubator/setup-miniconda@v2
- uses: conda-incubator/setup-miniconda@v3
with:
auto-update-conda: true
activate-environment: esmvaltool-fromlock
python-version: "3.11"
python-version: "3.12"
miniforge-version: "latest"
miniforge-variant: Mambaforge
use-mamba: true
Expand All @@ -38,7 +38,7 @@ jobs:
conda list
conda config --show-sources
conda config --show
# setup-miniconda@v2 installs an old conda and mamba
# setup-miniconda@v3 installs an old conda and mamba
# forcing a modern mamba updates both mamba and conda
conda install -c conda-forge "mamba>=1.4.8"
conda --version
Expand Down Expand Up @@ -71,8 +71,6 @@ jobs:
run: |
esmvaltool --help
esmvaltool version
- name: Run flake8
run: flake8
- name: Run pytests
run: pytest -n 2 -m "not installation"
# Automated PR
Expand Down
14 changes: 8 additions & 6 deletions .github/workflows/install-from-conda.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@ jobs:
runs-on: "ubuntu-latest"
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]
# fail-fast set to False allows all other tests
# in the workflow to run regardless of any fail
fail-fast: false
name: Linux Python ${{ matrix.python-version }}
steps:
- uses: conda-incubator/setup-miniconda@v2
- uses: conda-incubator/setup-miniconda@v3
with:
python-version: ${{ matrix.python-version }}
miniforge-version: "latest"
Expand All @@ -66,7 +66,7 @@ jobs:
esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
- name: Upload artifacts
if: ${{ always() }} # upload artifacts even if fail
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: Conda_Install_Linux_python_${{ matrix.python-version }}
path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
Expand All @@ -75,12 +75,14 @@ jobs:
runs-on: "macos-latest"
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]
architecture: ["x64"] # need to force Intel, arm64 builds have issues
fail-fast: false
name: OSX Python ${{ matrix.python-version }}
steps:
- uses: conda-incubator/setup-miniconda@v2
- uses: conda-incubator/setup-miniconda@v3
with:
architecture: ${{ matrix.architecture }}
python-version: ${{ matrix.python-version }}
miniforge-version: "latest"
miniforge-variant: Mambaforge
Expand All @@ -100,7 +102,7 @@ jobs:
esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
- name: Upload artifacts
if: ${{ always() }} # upload artifacts even if fail
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: Conda_Install_OSX_python_${{ matrix.python-version }}
path: conda_install_osx_artifacts_python_${{ matrix.python-version }}
9 changes: 4 additions & 5 deletions .github/workflows/install-from-condalock-file.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ jobs:
runs-on: "ubuntu-latest"
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]
fail-fast: false
name: Linux Python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: conda-incubator/setup-miniconda@v2
- uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: esmvaltool-fromlock
python-version: ${{ matrix.python-version }}
Expand All @@ -51,11 +51,10 @@ jobs:
- run: pip install -e .[develop]
- run: esmvaltool --help
- run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
- run: flake8
- run: pytest -n 2 -m "not installation"
- name: Upload artifacts
if: ${{ always() }} # upload artifacts even if fail
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: Source_Install_Linux_python_${{ matrix.python-version }}
path: source_install_linux_artifacts_python_${{ matrix.python-version }}
18 changes: 10 additions & 8 deletions .github/workflows/install-from-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,14 @@ jobs:
runs-on: "ubuntu-latest"
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]
# fail-fast set to False allows all other tests
# in the workflow to run regardless of any fail
fail-fast: false
name: Linux Python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- uses: conda-incubator/setup-miniconda@v2
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: esmvalcore
environment-file: environment.yml
Expand All @@ -68,7 +68,7 @@ jobs:
esmvaltool version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
- name: Upload artifacts
if: ${{ always() }} # upload artifacts even if fail
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: PIP_Install_Linux_python_${{ matrix.python-version }}
path: pip_install_linux_artifacts_python_${{ matrix.python-version }}
Expand All @@ -77,13 +77,15 @@ jobs:
runs-on: "macos-latest"
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]
architecture: ["x64"] # need to force Intel, arm64 builds have issues
fail-fast: false
name: OSX Python ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v3
- uses: conda-incubator/setup-miniconda@v2
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
with:
architecture: ${{ matrix.architecture }}
activate-environment: esmvalcore
environment-file: environment.yml
python-version: ${{ matrix.python-version }}
Expand All @@ -104,7 +106,7 @@ jobs:
esmvaltool version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
- name: Upload artifacts
if: ${{ always() }} # upload artifacts even if fail
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v4
with:
name: PIP_Install_OSX_python_${{ matrix.python-version }}
path: pip_install_osx_artifacts_python_${{ matrix.python-version }}
Loading

0 comments on commit 99def59

Please sign in to comment.