diff --git a/.circleci/config.yml b/.circleci/config.yml index d42e6ada13..9765cf6c40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,8 +2,8 @@ version: 2.1 orbs: - coverage-reporter: codacy/coverage-reporter@11.4.1 - codecov: codecov/codecov@1.2.3 + coverage-reporter: codacy/coverage-reporter@13.13.7 + codecov: codecov/codecov@3.2.5 commands: check_changes: @@ -32,7 +32,7 @@ commands: pytest -n 4 --junitxml=test-reports/report.xml esmvaltool version - store_test_results: - path: test-reports/ + path: test-reports/report.xml - store_artifacts: path: /logs - run: @@ -124,6 +124,9 @@ jobs: - /root/.cache/pip - .mypy_cache - .pytest_cache + - run: + name: Install gpg (required by codecov orb) + command: apt update && apt install -y gpg - codecov/upload: when: always file: 'test-reports/coverage.xml' diff --git a/.github/workflows/run-tests-comment.yml b/.github/workflows/run-tests-comment.yml deleted file mode 100644 index ccae9db0ee..0000000000 --- a/.github/workflows/run-tests-comment.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Test via PR Comment -on: - issue_comment: - types: - - created - -# Required shell entrypoint to have properly configured bash shell -defaults: - run: - shell: bash -l {0} - -jobs: - linux: - runs-on: "ubuntu-latest" - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - fail-fast: false - name: Linux Python ${{ matrix.python-version }} - if: ${{ github.event.issue.pull_request && github.event.comment.body == '@runGAtests' }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: esmvalcore - environment-file: environment.yml - python-version: ${{ matrix.python-version }} - miniforge-version: "latest" - miniforge-variant: Mambaforge - use-mamba: true - - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }} - - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt - - run: python -V 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt - - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 - - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - - name: Upload artifacts - if: ${{ always() }} # upload artifacts even if fail - uses: actions/upload-artifact@v2 - with: - name: Test_Linux_python_${{ matrix.python-version }} - path: test_linux_artifacts_python_${{ matrix.python-version }} - - osx: - runs-on: "macos-latest" - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - fail-fast: false - name: OSX Python ${{ matrix.python-version }} - if: ${{ github.event.issue.pull_request && github.event.comment.body == '@runGAtests' }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: esmvalcore - environment-file: environment.yml - python-version: ${{ matrix.python-version }} - miniforge-version: "latest" - miniforge-variant: Mambaforge - use-mamba: true - - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }} - - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt - - run: python -V 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt - - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - - run: flake8 - - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - - name: Upload artifacts - if: ${{ always() }} # upload artifacts even if fail - uses: actions/upload-artifact@v2 - with: - name: Test_OSX_python_${{ matrix.python-version }} - path: test_osx_artifacts_python_${{ matrix.python-version }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c95c86a3fc..2e4d195c1c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,9 +14,11 @@ build: jobs: pre_create_environment: # update mamba just in case - - mamba update --yes --quiet --name=base mamba + - mamba update --yes --quiet --name=base mamba 'zstd=1.5.2' - mamba --version + - mamba list --name=base post_create_environment: + - conda run -n ${CONDA_DEFAULT_ENV} mamba list # use conda run executable wrapper to have all env variables - conda run -n ${CONDA_DEFAULT_ENV} mamba --version - conda run -n ${CONDA_DEFAULT_ENV} pip install . --no-deps diff --git a/CITATION.cff b/CITATION.cff index 1f5a31582f..70cf872768 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -201,11 +201,11 @@ authors: given-names: Joerg cff-version: 1.2.0 -date-released: 2023-07-04 +date-released: 2023-11-01 doi: "10.5281/zenodo.3387139" license: "Apache-2.0" message: "If you use this software, please cite it using these metadata." repository-code: "https://github.com/ESMValGroup/ESMValCore/" title: ESMValCore -version: "v2.9.0" +version: "v2.10.0rc1" ... diff --git a/README.md b/README.md index 5b51d55cf1..20d13f1637 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ [![codecov](https://codecov.io/gh/ESMValGroup/ESMValCore/branch/main/graph/badge.svg?token=wQnDzguwq6)](https://codecov.io/gh/ESMValGroup/ESMValCore) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/5d496dea9ef64ec68e448a6df5a65783)](https://www.codacy.com/gh/ESMValGroup/ESMValCore?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValCore&utm_campaign=Badge_Grade) [![Docker Build Status](https://img.shields.io/docker/cloud/build/esmvalgroup/esmvalcore)](https://hub.docker.com/r/esmvalgroup/esmvalcore/) -[![Anaconda-Server Badge](https://anaconda.org/conda-forge/esmvalcore/badges/version.svg)](https://anaconda.org/conda-forge/esmvalcore) +[![Anaconda-Server Badge](https://img.shields.io/badge/Anaconda.org-2.9.0-blue.svg)](https://anaconda.org/conda-forge/esmvalcore) [![Github Actions Test](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml) -[![Github Actions Dashboard](https://api.meercode.io/badge/ESMValGroup/ESMValCore?type=ci-success-rate&branch=main&lastDay=14)](https://meercode.io) ![esmvaltoollogo](https://raw.githubusercontent.com/ESMValGroup/ESMValCore/main/doc/figures/ESMValTool-logo-2.png) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index cda69cb7f4..69c8d6ab36 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 2b5ce1d2957de54bf44f92cef302051df5f4407ef5d8d75d24f94604e083413f +# input_hash: 55d83fc9b0296f7958ba5243657df8702c78bc2ffca2c410e5ded6c9b712abdf @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.7.22-hbcca054_0.conda#a73ecd2988327ad4c8f2c331482917f2 @@ -9,70 +9,68 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed3 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2#19410c3df09dfb12d1206132a1d357c5 https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_16.conda#7ca122655873935e02c91279c5b03c8c -https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.39-hcc3a1bd_1.conda#737be0d34c22d24432049ab7a3214de4 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-12.2.0-h3b97bd3_19.tar.bz2#199a7292b1d3535376ecf7670c231d1f -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.1.0-h15d22d2_0.conda#afb656a334c409dd9805508af1c89c7a -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-12.2.0-h3b97bd3_19.tar.bz2#277d373b57791ee71cafc3c5bfcf0641 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.1.0-hfd8a6a1_0.conda#067bcc23164642f4c226da631f2a2e1d +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda#7aca3059a1729aa76c597603f10b0dd3 +https://conda.anaconda.org/conda-forge/linux-64/libboost-headers-1.82.0-ha770c72_6.conda#a943dcb8fd22cf23ce901ac84f6538c2 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-12.3.0-h8bca6fd_2.conda#ed613582de7b8569fdc53ca141be176a +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-12.3.0-h8bca6fd_2.conda#7268a17e56eb099d1b8869bbbf46de4c +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_2.conda#9172c297304f2a20134fc56c97fbe229 https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda#d8d7293c5b37f39b2ac32940621c6592 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-3_cp311.conda#c2e2630ddb68cf52eec74dc7dfab20b5 +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-4_cp311.conda#d786502c97404c94d7d58d258a445a65 https://conda.anaconda.org/conda-forge/noarch/tzdata-2023c-h71feb2d_0.conda#939e3e74d8be4dac89ce83b20de2492a https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.1.0-h69a702a_0.conda#506dc07710dd5b0ba63cbf134897fc10 -https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.1.0-he5830b7_0.conda#56ca14d57ac29a75d23a39eb3ee0ddeb +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_2.conda#e2042154faafe61969556f28bade94b9 https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.12-he073ed8_16.conda#071ea8dceff4d30ac511f4a2f8437cd1 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d -https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.39-he00db2b_1.conda#3d726e8b51a1f5bfd66892a2b7d9db2d +https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.40-hf600244_0.conda#33084421a8c0af6aef1b439707f7662a https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab -https://conda.anaconda.org/conda-forge/linux-64/binutils-2.39-hdd6e379_1.conda#1276c18b0a562739185dbf5bd14b57b2 -https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.39-h5fc0e48_13.conda#7f25a524665e4e2f8a5f86522f8d0e31 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.1.0-he5830b7_0.conda#cd93f779ff018dd85c7544c015c9db3c -https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.8.23-hd590300_0.conda#cc4f06f7eedb1523f3b83fd0fb3942ff +https://conda.anaconda.org/conda-forge/linux-64/binutils-2.40-hdd6e379_0.conda#ccc940fddbc3fcd3d79cd4c654c4b5c4 +https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.40-hbdbef99_2.conda#adfebae9fdc63a598495dfe3b006973a +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_2.conda#c28003b0be0494f9a7664389146716ff +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.4-hd590300_0.conda#8dacaf703f8e57aa0c4f0c5c8f4be39b https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54 -https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.19.1-hd590300_0.conda#e8c18d865be43e2fb3f7a145b6adf1f5 -https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h166bdaf_1.tar.bz2#897e772a157faf3330d72dd291486f62 +https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.20.1-hd590300_1.conda#2facbaf5ee1a56967aecaee89799160e https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2#ac7bc6a654f8f41b352b38f4051135f8 -https://conda.anaconda.org/conda-forge/linux-64/geos-3.11.2-hcb278e6_0.conda#3b8e364995e3575e57960d29c1e5ab14 +https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.0-h59595ed_0.conda#3fdf79ef322c8379ae83be491d805369 https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2#14947d8770185e5153fdd04d4673ed37 https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-he1b5a44_1004.tar.bz2#cddaf2c63ea4a5901cf09524c490ecdc https://conda.anaconda.org/conda-forge/linux-64/giflib-5.2.1-h0b41bf4_3.conda#96f3b11872ef6fad973eac856cd2624f https://conda.anaconda.org/conda-forge/linux-64/gmp-6.2.1-h58526e2_0.tar.bz2#b94cf2db16066b242ebd26db2facbd56 https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h58526e2_1001.tar.bz2#8c54672728e8ec6aa6db90cf2806d220 -https://conda.anaconda.org/conda-forge/linux-64/icu-72.1-hcb278e6_0.conda#7c8d20d847bb45f56bd941578fcfa146 -https://conda.anaconda.org/conda-forge/linux-64/json-c-0.16-hc379101_0.tar.bz2#0e2bca6857cb73acec30387fef7c3142 +https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda#cc47e1facc155f91abd89b11e48e72ff +https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda#9961b1f100c3b6852bd97c9233d06979 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f -https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230125.3-cxx17_h59595ed_0.conda#d1db1b8be7c3a8983dcbbbfe4f0765de -https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda#0f683578378cddb223e7fd24f785ab2a -https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.0.9-h166bdaf_9.conda#61641e239f96eae2b8492dc7e755828c +https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230802.1-cxx17_h59595ed_0.conda#2785ddf4cb0e7e743477991d64353947 +https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.2-h59595ed_1.conda#127b0be54c1c90760d7fe02ea7a56426 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_1.conda#aec6c91c7371c26392a06708a73c70e5 https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 -https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.18-h0b41bf4_0.conda#6aa9c9de5542ecb07fdda9ca626252d8 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.19-hd590300_0.conda#1635570038840ee3f9c71d22aa5b8b6d https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_2.conda#78fdab09d9138851dde2b5fe2a11019e https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d -https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-h0b41bf4_0.conda#1edd9e67bdb90d78cea97733ff6b54e6 -https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2#39b1328babf85c7c3a61636d9cd50206 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.0.0-hd590300_1.conda#ea25936bb4080d843790b586850f82b8 +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 https://conda.anaconda.org/conda-forge/linux-64/libnuma-2.0.16-h0b41bf4_1.conda#28bfe2cb11357ccc5be21101a6b7ce86 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.23-pthreads_h80387f5_0.conda#9c5ea51ccb8ffae7d06c645869d24ce6 -https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.2.0-h46fd767_19.tar.bz2#80d0e00150401e9c06a055f36e8e73f2 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.3.0-h0f45ef3_2.conda#4655db64eca78a6fcc4fb654fc1f8d57 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.18-h36c2ea0_1.tar.bz2#c3788462a6fbddafdb413a9f9053e58d https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda#f204c8ba400ec475452737094fb81d52 https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar.bz2#ede4266dc02e875fe1ea77b25dd43747 https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b -https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.1-hd590300_0.conda#82bf6f63eb15ef719b556b63feec3a77 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.2-hd590300_0.conda#30de3fd9b3b602f7473f30e684eeea8c https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0 https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2#bb14fcb13341b81d5eb386423b9d2bac -https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda#681105bccc2a3f7f1a837d47d39c9179 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-h59595ed_2.conda#7dbaa197d7ba6032caf7ae7f32c1efa0 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.2-hd590300_0.conda#e5ac5227582d6c83ccf247288c0eb095 -https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2#660e72c82f2e75a6b3fe6a6e75c79f19 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.4-hd590300_0.conda#412ba6938c3e2abaca8b1129ea82e238 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.42.2-h59595ed_0.conda#700edd63ccd5fc66b70b1c028cea9a68 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 https://conda.anaconda.org/conda-forge/linux-64/rdma-core-28.9-h59595ed_1.conda#aeffb7c06b5f65e55e6c637408dc4100 -https://conda.anaconda.org/conda-forge/linux-64/re2-2023.03.02-h8c504da_0.conda#206f8fa808748f6e90599c3368a1114e https://conda.anaconda.org/conda-forge/linux-64/snappy-1.1.10-h9fff704_0.conda#e6d228cd0bb74a51dd18f5bfce0b4115 https://conda.anaconda.org/conda-forge/linux-64/tzcode-2023c-h0b41bf4_0.conda#0c0533894f21c3d35697cb8378d390e2 +https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.7-hcb278e6_1.conda#2c46deb08ba9b10e90d0a6401ad65deb https://conda.anaconda.org/conda-forge/linux-64/xorg-kbproto-1.0.7-h7f98852_1002.tar.bz2#4b230e8381279d76131116660f5a241a https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hd590300_0.conda#b462a33c0be1421532f28bfe8f4a7514 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.11-hd590300_0.conda#2c80dc38fface310c9bd81b17037fee5 @@ -80,139 +78,141 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.3-h7f98852_0.t https://conda.anaconda.org/conda-forge/linux-64/xorg-renderproto-0.11.1-h7f98852_1002.tar.bz2#06feff3d2634e3097ce2fe681474b534 https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-h0b41bf4_1003.conda#bce9f945da8ad2ae9b1d7165a64d0f87 https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007.tar.bz2#b4a4381d54784606820704f7b5f05a15 -https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.1-h0b41bf4_0.conda#e9c3bcf0e0c719431abec8ca447eee27 +https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.2-hd590300_0.conda#f08fb5c89edfc4aadee1c81d4cfb1fa1 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.0-h93469e0_0.conda#580a52a05f5be28ce00764149017c6d4 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h862ab75_1.conda#0013fcee7acb3cfc801c5929824feb3c -https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h862ab75_0.conda#be7020c516cdf1e5937bc56e37aca7ca -https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.16-h862ab75_1.conda#f883d61afbc95c50f7b3f62546da4235 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.7-h6e18cf3_0.conda#cdbd44927a53a313d69f3c206a418dd2 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h037bafe_4.conda#72cb3661f349a95ea48b0ddcdc4c0f18 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h037bafe_3.conda#6c2ea725535e0f2a18f645a0bf03a8f6 +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h037bafe_3.conda#ac1b0e60de127cc46a04e76a907434a1 https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda#8b9b5aca60558d02ddaa09d599e55920 -https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.2.0-hcc96c02_19.tar.bz2#bb48ea333c8e6dcc159a1575f04d869e +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.3.0-he2b93b0_2.conda#2f4d8677dc7dd87f93e9abfb2ce86808 https://conda.anaconda.org/conda-forge/linux-64/glog-0.6.0-h6f12383_0.tar.bz2#b31f3565cb84435407594e548a2fb7b2 -https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h501b40f_6.conda#c3e9338e15d90106f467377017352b97 -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-17_linux64_openblas.conda#57fb44770b1bc832fb2dbefa1bd502de -https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.0.9-h166bdaf_9.conda#081aa22f4581c08e4372b0b6c2f8478e -https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.0.9-h166bdaf_9.conda#1f0a03af852a9659ed2bf08f2f1704fd +https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h2a13503_7.conda#bd77f8da987968ec3927990495dc22e4 +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_1.conda#f07002e225d7a60a694d42a7bf5ff53f +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_1.conda#5fc11c6020d421960607d821310fcd4d https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d -https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.52.0-h61bc06f_0.conda#613955a50485812985c059e7b269f42e +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_2.conda#e75a75a6eaf6f318dae2631158c46575 +https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h01aab08_1018.conda#3eb5f16bcc8a02892199aa63555c731f +https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.55.1-h47da74e_0.conda#a802251d1eaeeae041c867faf0f94fa8 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda#e1c890aebdebbfbf87e2c917187b4416 -https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.23.3-hd1fb520_0.conda#c8da7f04073ed0fabcb60885a4c1a722 -https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-h0d5128d_13.conda#e1d6139ff0500977a760567a4bec1ce9 -https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.42.0-h2797004_0.conda#fdaae20a1cf7cd62130a0973190a31b7 +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.24.3-hf27288f_1.conda#5097789a2bc83e697d7509df57f25bfd +https://conda.anaconda.org/conda-forge/linux-64/libre2-11-2023.06.02-h7a70373_0.conda#c0e7eacd9694db3ef5ef2979a7deea70 +https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hb58d41b_14.conda#264f9a3a4ea52c8f4d3e8ae1213a3335 +https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.44.0-h2797004_0.conda#b58e6816d137f3aabf77d341dd5d732b https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda#1f5a58e686b13bcfde88b93f547d23fe +https://conda.anaconda.org/conda-forge/linux-64/libudunits2-2.2.28-h40f5838_3.conda#4bdace082e911a3e1f1f0b721bed5b56 https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda#33277193f5b92bad9fdd230eb700929c -https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h0d562d8_0.conda#558ab736404275d7df61c473c1af35aa -https://conda.anaconda.org/conda-forge/linux-64/libzip-1.9.2-hc929e4a_1.tar.bz2#5b122b50e738c4be5c3f2899f010d7cf +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h232c23b_1.conda#f3858448893839820d4bcfb14ad3ecdf +https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_3.conda#ac79812548e7e8cf61f7b0abdef01d3b https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2#69e2c796349cd9b273890bee0febfe1b https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 -https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.46-h06160fa_0.conda#413d96a0b655c8f8aacc36473a2dbb04 -https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2#5b8c42eb62e9fc961af70bdd6a26e168 -https://conda.anaconda.org/conda-forge/linux-64/ucx-1.14.1-hf587318_2.conda#37b27851c8d5a9a98e61ecd36aa990a7 +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.55-h06160fa_0.conda#8cdfb7d58bdfd543717eeacc0801f3c0 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-h2797004_0.conda#513336054f884f95d9fd925748f41ef3 +https://conda.anaconda.org/conda-forge/linux-64/ucx-1.15.0-h64cca9d_0.conda#b35b1f1a9fdbf93266c91f297dc9060e https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda#93ee23f12bc2e684548181256edd2cf6 -https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.4-h9c3ff4c_1.tar.bz2#21743a8d2ea0c8cfbbf8fe489b0347df +https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h59595ed_0.conda#8851084c192dbc56215ac4e3c9aa30fa https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda#68c34ec6149623be41a1933ab996a209 -https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.2-hfc55251_7.conda#32ae18eb2a687912fc9e92a501c0a11b -https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.28-h3870b5a_0.conda#b775667301ab249f94ad2bea91fc4223 -https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.4-h0f2a231_0.conda#876286b5941933a0f558777e57d883cc -https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h6582d0a_3.conda#d3c3c7698d0b878aab1b86db95407c8e -https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.0.9-h166bdaf_9.conda#d47dee1856d9cb955b8076eeff304a5b -https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda#e1232042de76d24539a436d37597eb06 -https://conda.anaconda.org/conda-forge/linux-64/gcc-12.2.0-h26027b1_13.conda#ec93d13e0fe8514f65842120dbae1b16 -https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.2.0-h4798a0e_13.conda#1d009211292e0d869a31f3bc5b4ee78b -https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.2.0-h55be85b_19.tar.bz2#143d770a2a2911cd84b98286db0e6a40 -https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.2.0-hcc96c02_19.tar.bz2#698aae34e4f5e0ea8eac0d529c8f20b6 -https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.1-h659d440_0.conda#1b5126ec25763eb17ef74c8763d26e84 -https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.6.2-h039dbb9_1.conda#29cf970521d30d113f3425b84cb250f6 -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-17_linux64_openblas.conda#7ef0969b00fe3d6eef56a8151d3afb29 -https://conda.anaconda.org/conda-forge/linux-64/libglib-2.76.4-hebfc3b9_0.conda#c6f951789c888f7bbd2dd6858eab69de -https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.56.2-h3905398_0.conda#a87e780f3d9cc7cf432e47ced83a67ce -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-17_linux64_openblas.conda#a2103882c46492e26500fcb56c03de8b -https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.18.1-h8fd135c_2.conda#bbf65f7688512872f063810623b755dc -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.1-h8b53f26_0.conda#8ad377fb60abab446a9f02c62b3c2190 +https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda#04b88013080254850d6c01ed54810589 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.35-hd1885a1_4.conda#a0728c6591063bee78f037741d1da83b +https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda#009521b7ed97cca25f8f997f9e745976 +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_1.conda#39f910d205726805a958da408ca194ba +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda#9ae35c3d96db2c94ce0cef86efdfa2cb +https://conda.anaconda.org/conda-forge/linux-64/gcc-12.3.0-h8d2909c_2.conda#e2f2f81f367e14ca1f77a870bda2fe59 +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.3.0-h76fc315_2.conda#11517e7b5c910c5b5d6985c0c7eb7f50 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.3.0-hfcedea8_2.conda#09d48cadff6669068c3bf7ae7dc8ea4a +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.3.0-he2b93b0_2.conda#f89b9916afc36fc5562fbfc11330a8a2 +https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda#cd95826dbd331ed1be26bdf401432844 +https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.2-h039dbb9_0.conda#611d6c83d1130ea60c916531adfb11db +https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.0-hebfc3b9_0.conda#e618003da3547216310088478e475945 +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 +https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.19.0-hb90f79a_1.conda#8cdb7d41faa0260875ba92414c487e2d +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.6.0-ha9c0a0a_2.conda#55ed21669b2015f77c180feb1dd41930 https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h0054252_1.conda#f27960e8873abb5476e96ef33bdbdccd -https://conda.anaconda.org/conda-forge/linux-64/nss-3.89-he45b914_0.conda#2745719a58eeaab6657256a3f142f099 -https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h385abfd_1.conda#2cd5aac7ef1b4c6ac51bf521251a89b3 +https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.2-h0ab5242_0.conda#c6eafb51b60db59fd2132f6bbaada9b3 +https://conda.anaconda.org/conda-forge/linux-64/nss-3.94-h1d7d5a4_0.conda#7caef74bbfa730e014b20f0852068509 +https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h208142c_3.conda#f983ae19192439116ca5b5589560f167 https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.1.3-h32600fe_0.conda#8287aeb8462e2d4b235eff788e75919d -https://conda.anaconda.org/conda-forge/linux-64/python-3.11.4-hab00c5b_0_cpython.conda#1c628861a2a126b9fc9363ca1b7d014e -https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.42.0-h2c6b66d_0.conda#1192f6ec654a5bc4ee1d64bdc4a3e5cc -https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-hc3e0081_0.tar.bz2#d4c341e0379c31e9e781d4f204726867 -https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.6-h8ee46fc_0.conda#7590b76c3d11d21caa44f3fc38ac584a +https://conda.anaconda.org/conda-forge/linux-64/python-3.11.6-hab00c5b_0_cpython.conda#b0dfbe2fcbfdb097d321bfd50ecddab1 +https://conda.anaconda.org/conda-forge/linux-64/re2-2023.06.02-h2873b5e_0.conda#bb2d5e593ef13fe4aff0bc9440f945ae +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.44.0-h2c6b66d_0.conda#df56c636df4a98990462d66ac7be2330 +https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_3.conda#6bb8deb138f87c9d48320ac21b87e7a1 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.7-h8ee46fc_0.conda#49e482d882669206653b095f5206c05b https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.13-pyhd8ed1ab_0.conda#06006184e203b61d3525f90de394471e -https://conda.anaconda.org/conda-forge/linux-64/antlr-python-runtime-4.7.2-py311h38be061_1003.tar.bz2#0ab8f8f0cae99343907fe68cda11baea +https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.11.1-pyhd8ed1ab_0.tar.bz2#15109c4977d39ad7aa3423f57243e286 https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz2#6c72ec3e660a51736913ef6ea68c454b https://conda.anaconda.org/conda-forge/noarch/attrs-23.1.0-pyh71513ae_1.conda#3edfead7cedd1ab4400a6c588f3e75f8 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.1-h9599702_1.conda#a8820ce2dbe6f7d54f6540d9a3a0028a -https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.11-hbe98c3e_0.conda#067641478d8f706b80a5a434a22b82be -https://conda.anaconda.org/conda-forge/noarch/backcall-0.2.0-pyh9f0ad1d_0.tar.bz2#6006a6d08a3fa99268a2681c7fb55213 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.2-he4fbe49_4.conda#38da036c9d74d4d44f35e05474135f77 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.13-hbbfb9a7_7.conda#2c4c47d83a0e111799dda4059c88621d https://conda.anaconda.org/conda-forge/noarch/backports-1.0-pyhd8ed1ab_3.conda#54ca2e08b3220c148a1d8329c2678e02 -https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py311h38be061_7.tar.bz2#ec62b3c5b953cb610f5e2b09cd776caf -https://conda.anaconda.org/conda-forge/linux-64/brotli-1.0.9-h166bdaf_9.conda#4601544b4982ba1861fa9b9c607b2c06 -https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.0.9-py311ha362b79_9.conda#ced5340f5dc6cff43a80deac8d0e398f +https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py311h38be061_8.conda#5384590f14dfe6ccd02811236afc9f8e +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda#f27a24d46e3ea7b70a1f98e50c62508f +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_1.conda#cce9e7c3f1c307f2a5fb08a2922d6164 https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.6.0-hd590300_0.conda#ea6c792f792bdd7ae6e7e2dee32f0a48 https://conda.anaconda.org/conda-forge/noarch/certifi-2023.7.22-pyhd8ed1ab_0.conda#7f3dbc9179b4dde7da98dfb151d0ad22 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c -https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.2.0-pyhd8ed1ab_0.conda#313516e9a4b08b12dfb1e1cd390a96e3 -https://conda.anaconda.org/conda-forge/noarch/click-8.1.6-unix_pyh707e725_0.conda#64dbb3b205546691a61204d1cfb208e3 -https://conda.anaconda.org/conda-forge/noarch/cloudpickle-2.2.1-pyhd8ed1ab_0.conda#b325bfc4cff7d7f8a868f1f7ecc4ed16 -https://conda.anaconda.org/conda-forge/noarch/codespell-2.2.5-pyhd8ed1ab_0.conda#c73551c990f6e7e9c83cdb8bdbafdeb8 +https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.2-pyhd8ed1ab_0.conda#7f4a9e3fcff3f6356ae99244a014da6a +https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda#f3ad426304898027fc619827ff428eca +https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.0.0-pyhd8ed1ab_0.conda#753d29fe41bb881e4b9c004f0abf973f +https://conda.anaconda.org/conda-forge/noarch/codespell-2.2.6-pyhd8ed1ab_0.conda#a206349b7bb7475ae580f987cb425bdd https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2#3faab06a954c2a04039983f2c4a50d99 -https://conda.anaconda.org/conda-forge/noarch/cycler-0.11.0-pyhd8ed1ab_0.tar.bz2#a50559fad0affdbb33729a68669ca1cb -https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.0-py311hb755f60_0.conda#257dfede48699e2e6372528d08399e5a +https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_0.conda#5cd86562580f274031ede6aa6aa24441 +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.5-py311hb755f60_0.conda#25b42509a68f96e612534af3fe2cf033 https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.bz2#43afe5ab04e35e17ba28649471dd7364 https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2#961b3a227b437d82ad7054484cfa71b2 https://conda.anaconda.org/conda-forge/noarch/dill-0.3.7-pyhd8ed1ab_0.conda#5e4f3466526c52bc9af2d2353a1460bd https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.7-pyhd8ed1ab_0.conda#12d8aae6994f342618443a8f05c652a0 -https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_0.conda#207175b7d514d42f977ec505800d6824 +https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_2.conda#33f8066e53679dd4be2355fec849bf01 https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b https://conda.anaconda.org/conda-forge/noarch/entrypoints-0.4-pyhd8ed1ab_0.tar.bz2#3cf04868fee0a029769bd41f4b2fbf2d -https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.2-pyhd8ed1ab_0.conda#de4cb3384374e1411f0454edcf546cdb +https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.3-pyhd8ed1ab_0.conda#e6518222753f519e911e83136d2158d9 https://conda.anaconda.org/conda-forge/noarch/execnet-2.0.2-pyhd8ed1ab_0.conda#67de0d8241e1060a479e3c37793e26f9 -https://conda.anaconda.org/conda-forge/noarch/executing-1.2.0-pyhd8ed1ab_0.tar.bz2#4c1bc140e2be5c8ba6e3acab99e25c50 -https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.2-pyhd8ed1ab_0.conda#53522ec72e6adae42bd373ef58357230 +https://conda.anaconda.org/conda-forge/noarch/executing-2.0.1-pyhd8ed1ab_0.conda#e16be50e378d8a4533b989035b196ab8 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.13.1-pyhd8ed1ab_0.conda#0c1729b74a8152fde6a38ba0a2ab9f45 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda#0f69b688f52ff6da70bccb7ff7001d1d -https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.6.0-pyh1a96a4e_0.conda#50ea2067ec92dfcc38b4f07992d7e235 -https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h6b639ba_2.conda#ee8220db21db8094998005990418fe5b +https://conda.anaconda.org/conda-forge/linux-64/freexl-2.0.0-h743c826_0.conda#12e6988845706b2cfbc3bc35c9a61a95 +https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.10.0-pyhca7485f_0.conda#5b86cf1ceaaa9be2ec4627377e538db1 +https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h829c605_4.conda#252a696860674caf7a855e16f680d63a https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 -https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.2.0-h8acd90e_13.conda#ca62fa3fcd15a7d92194ee2ff7c9c54b -https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-12.2.0-h307d370_13.conda#d4a4dd80f5a470407eed32cb2932174a +https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.3.0-h499e0f7_2.conda#0558a8c44eb7a18e6682bd3a8ae6dcab +https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-12.3.0-h7fe76b4_2.conda#3a749210487c0358b6f135a648cbbf60 https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda#4d8df0b0db060d33c9a702ada998a8fe -https://conda.anaconda.org/conda-forge/linux-64/gxx-12.2.0-h26027b1_13.conda#de605ff437f3fdc010f1b529642339f1 -https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-12.2.0-hb41e900_13.conda#cf45659115653fc80623116ccd0a6ae2 -https://conda.anaconda.org/conda-forge/linux-64/humanfriendly-10.0-py311h38be061_4.tar.bz2#5c4f38a9e482f00a7bf23fe479c8ca29 +https://conda.anaconda.org/conda-forge/linux-64/gxx-12.3.0-h8d2909c_2.conda#673bac341be6b90ef9e8abae7e52ca46 +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-12.3.0-h8a814eb_2.conda#f517b1525e9783849bd56a5dc45a9960 +https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd8ed1ab_6.conda#2ed1fe4b9079da97c44cfe9c2e5078fd https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#34272b248891bddccc64479f9a7fffed https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda#f800d2da156d08e289b14e87e43c1ae5 https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.tar.bz2#3c3de74912f11d2b590184f03c7cd09b -https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py311h4dd048b_1.tar.bz2#46d451f575392c01dc193069bd89766d -https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h2582759_0.conda#07745544b144855ed4514a4cf0aadd74 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-haa2dc70_1.conda#980d8aca0bc23ca73fa8caa3e7c84c28 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.2.1-hca28451_0.conda#96aec6156d58591f5a4e67056521ce1b -https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 -https://conda.anaconda.org/conda-forge/linux-64/libpq-15.4-hfc447b1_0.conda#b9ce311e7aba8b5fc3122254f0a6e97e -https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.1-hbf2b3c1_0.conda#4963f3f12db45a576f2b8fbe9a0b8569 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py311h9547e67_1.conda#2c65bdf442b0d37aad080c8a4e0d452f +https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h459d7ec_1.conda#7cc99d87755a9e64586a6004c5f0f534 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-hb7c19ff_3.conda#e96637dd92c5f340215c753a5c9a22d7 +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-19_linux64_openblas.conda#420f4e9be59d0dc9133a0f43f7bab3f3 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.4.0-hca28451_0.conda#1158ac1d2613b28685644931f11ee807 +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.58.2-he06187c_0.conda#1eec35ecc7bd34e4ec2f2b4eccb8816e +https://conda.anaconda.org/conda-forge/linux-64/libpq-16.0-hfc447b1_1.conda#e4a9a5ba40123477db33e02a78dffb01 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.2-h658648e_1.conda#0ebb65e8d86843865796c7c95a941f34 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 -https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_0.conda#59a580306d62ef144c9dd592b5120f36 -https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py311h9f220a4_0.conda#b8aad2507303e04037e8d02d8ac54217 -https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py311h459d7ec_0.conda#9904dc4adb5d547cb21e136f98cb24b0 +https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_1.conda#aab51e50d994e58efdfa5382139b0468 +https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py311h38e4bf4_1.conda#f8e0b648d77bbe44d1fe8af8cc56a590 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py311h459d7ec_1.conda#71120b5155a0c500826cf81536721a15 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.7.0-pyhd8ed1ab_0.tar.bz2#34fc335fc50eef0b5ea708f2b5f54e0c -https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.0-pyhd8ed1ab_0.conda#c7d0ea64c37752ecbe6da458aee662d2 -https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.5-py311ha3edf6b_0.conda#7415f24f8c44e44152623d93c5015000 +https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.1-pyhd8ed1ab_0.conda#1dad8397c94e4de97a70de552a7dcf49 +https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.6-py311h9547e67_0.conda#e826b71bf3dc8c91ee097663e2bcface https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda#376b32e8f9d3eacbd625f37d39bd507d https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda#4eccaeba205f0aed9ac3a9ea58568ca3 -https://conda.anaconda.org/conda-forge/noarch/networkx-3.1-pyhd8ed1ab_0.conda#254f787d5068bc89f578bf63893ce8b4 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.25.2-py311h64a7726_0.conda#71fd6f1734a0fa64d8f852ae7156ec45 -https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda#5ce6a42505c6e9e6151c54c3ec8d68ea -https://conda.anaconda.org/conda-forge/noarch/packaging-23.1-pyhd8ed1ab_0.conda#91cda59e66e1e4afe9476f8ef98f5c30 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.2.1-pyhd8ed1ab_0.conda#425fce3b531bed6ec3c74fab3e5f0a1c +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h488ebb8_3.conda#128c25b7fe6a25286a48f3a6a9b5b6f3 +https://conda.anaconda.org/conda-forge/noarch/packaging-23.2-pyhd8ed1ab_0.conda#79002079284aa895f883c6b7f3f88fd6 https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9 https://conda.anaconda.org/conda-forge/noarch/parso-0.8.3-pyhd8ed1ab_0.tar.bz2#17a565a0c3899244e938cdf417e7b094 https://conda.anaconda.org/conda-forge/noarch/pathspec-0.11.2-pyhd8ed1ab_0.conda#e41debb259e68490e3ab81e46b639ab6 https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-py_1003.tar.bz2#415f0ebb6198cc2801c73438a9fb5761 -https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_0.tar.bz2#89e3c7cdde7d3aaa2aee933b604dd07f -https://conda.anaconda.org/conda-forge/noarch/pluggy-1.2.0-pyhd8ed1ab_0.conda#7263924c642d22e311d9e59b839f1b33 -https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h2582759_0.conda#a90f8e278c1cd7064b2713e6b7db87e6 +https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_1.conda#405678b942f2481cecdb3e010f4925d9 +https://conda.anaconda.org/conda-forge/noarch/pluggy-1.3.0-pyhd8ed1ab_0.conda#2390bd10bed1f3fdc7a537fb5a447d8d +https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h459d7ec_1.conda#490d7fa8675afd1aa6f1b2332d156a45 https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd3deb0d_0.tar.bz2#359eeb6536da0e687af562ed265ec263 https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.2-pyhd8ed1ab_0.tar.bz2#6784285c7e55cb7212efabc79e4c2883 https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 @@ -220,210 +220,211 @@ https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.9.1-pyhd8ed1ab_0.tar https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2#076becd9e05608f8dc72757d5f3a91ff https://conda.anaconda.org/conda-forge/noarch/pyflakes-2.5.0-pyhd8ed1ab_0.tar.bz2#1b3bef4313288ae8d35b1dfba4cd84a3 https://conda.anaconda.org/conda-forge/noarch/pygments-2.16.1-pyhd8ed1ab_0.conda#40e5cb18165466773619e5c963f00a7b -https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.bz2#e8fbc1b54b25f4b08281467bc13b70cc +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.1.1-pyhd8ed1ab_0.conda#176f7d56f0cfe9008bdf1bccd7de02fb https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#92a889dc236a5197612bc85bee6d7174 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 -https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.18.0-pyhd8ed1ab_0.conda#3be9466311564f80f8056c0851fc5bb7 +https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.18.1-pyhd8ed1ab_0.conda#305141cff54af2f90e089d868fffce28 https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2023.3-pyhd8ed1ab_0.conda#2590495f608a63625e165915fb4e2e34 -https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.2.0-py311h2582759_0.conda#dfcc3e6e30d6ec2b2bb416fcd8ff4dc1 -https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3-pyhd8ed1ab_0.conda#d3076b483092a435832603243567bc31 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0-py311hd4cff14_5.tar.bz2#da8769492e423103c59f469f4f17f8d9 -https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h75c88c4_0.conda#af6d43afe0d179ac83b7e0c16b2caaad -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.9.2-py311h46250e7_0.conda#bcf66b5abaec47198b42cdd0bb968540 -https://conda.anaconda.org/conda-forge/noarch/semver-3.0.1-pyhd8ed1ab_0.conda#ed90854ae56fb6edae1f13b4663b21b0 +https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.4.1-py311h459d7ec_0.conda#60b5332b3989fda37884b92c7afd6a91 +https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3.post1-pyhd8ed1ab_0.conda#c93346b446cd08c169d843ae5fc0da97 +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_1.conda#52719a74ad130de8fb5d047dc91f247a +https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h34ded2d_2.conda#ea365280db99687905b4d76cf6a3568c +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.6-py311h46250e7_0.conda#b7237977dbca779aaaab65f42f5558d9 +https://conda.anaconda.org/conda-forge/noarch/semver-3.0.2-pyhd8ed1ab_0.conda#5efb3fccda53974aed800b6d575f72ed https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df -https://conda.anaconda.org/conda-forge/noarch/setuptools-68.0.0-pyhd8ed1ab_0.conda#5a7739d0f57ee64133c9d32e6507c46d +https://conda.anaconda.org/conda-forge/noarch/setuptools-68.2.2-pyhd8ed1ab_0.conda#fc2166155db840c634a1291a5c35a709 https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 -https://conda.anaconda.org/conda-forge/noarch/smmap-3.0.5-pyh44b312d_0.tar.bz2#3a8dc70789709aa315325d5df06fb7e4 +https://conda.anaconda.org/conda-forge/noarch/smmap-5.0.0-pyhd8ed1ab_0.tar.bz2#62f26a3d1387acee31322208f0cfa3e0 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2#4d22a9315e78c6827f806065957d566e https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_0.tar.bz2#6d6552722448103793743dabfbda532d -https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.3.2.post1-pyhd8ed1ab_0.tar.bz2#146f4541d643d48fc8a75cacf69f03ae +https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.5-pyhd8ed1ab_1.conda#3f144b2c34f8cb5a9abd9ed23a39c561 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_0.conda#da1d979339e2714c30a8e806a33ec087 https://conda.anaconda.org/conda-forge/noarch/sqlparse-0.4.4-pyhd8ed1ab_0.conda#2e2f31b3b1c866c29636377e14f8c4c6 -https://conda.anaconda.org/conda-forge/noarch/tblib-1.7.0-pyhd8ed1ab_0.tar.bz2#3d4afc31302aa7be471feb6be048ed76 +https://conda.anaconda.org/conda-forge/noarch/tblib-2.0.0-pyhd8ed1ab_0.conda#f5580336fe091d46f9a2ea97da044550 https://conda.anaconda.org/conda-forge/noarch/termcolor-2.3.0-pyhd8ed1ab_0.conda#440d508f025b1692168caaf436504af3 https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.12.1-pyha770c72_0.conda#62f5b331c53d73e2f6c4c130b53518a0 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 -https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.2-py311h459d7ec_0.conda#12b1c374ee90a1aa11ea921858394dc8 -https://conda.anaconda.org/conda-forge/noarch/traitlets-5.9.0-pyhd8ed1ab_0.conda#d0b4f5c87cd35ac3fb3d47b223263a64 -https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.11-pyhd8ed1ab_0.conda#22776dce28e8ba933e5cbcf20b62c583 -https://conda.anaconda.org/conda-forge/noarch/types-urllib3-1.26.25.14-pyhd8ed1ab_0.conda#06118f39abab2ab953276a50b2775509 -https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.7.1-pyha770c72_0.conda#c39d6a09fe819de4951c2642629d9115 -https://conda.anaconda.org/conda-forge/linux-64/ujson-5.7.0-py311hcafe171_0.conda#ec3960b6d13bb60aad9c67f42a801720 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_1.conda#a700fcb5cedd3e72d0c75d095c7a6eda +https://conda.anaconda.org/conda-forge/noarch/traitlets-5.13.0-pyhd8ed1ab_0.conda#8a9953c15e1e5a7c1baddbbf4511a567 +https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.12-pyhd8ed1ab_0.conda#0cb14c80f66937df894d60626dd1921f +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.8.0-pyha770c72_0.conda#5b1be40a26d10a06f6d4f1f9e19fa0c7 +https://conda.anaconda.org/conda-forge/linux-64/ujson-5.8.0-py311hb755f60_0.conda#91e67c62c48444e4efc08fb61835abe8 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-py_0.tar.bz2#1447ead40f2a01733a9c8dfc32988375 -https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2#3563be4c5611a44210d9ba0c16113136 +https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 https://conda.anaconda.org/conda-forge/noarch/webob-1.8.7-pyhd8ed1ab_0.tar.bz2#a8192f3585f341ea66c60c189580ac67 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.1-pyhd8ed1ab_0.conda#8f467ba2db2b5470d297953d9c1f9c7d -https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h2582759_0.conda#15565d8602a78c6a994e4d9fcb391920 +https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.3-pyhd8ed1ab_0.conda#3fc026b9c87d091c4b34a6c997324ae8 +https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h459d7ec_1.conda#f4d770a09066aaa313b5cc22c0f6e9d1 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda#82b6df12252e6f32402b96dacc656fec https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda#ed67c36f215b310412b2af935bf3e530 -https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.7.0-pyhd8ed1ab_0.conda#aacae3c0eaba0204dc6c5497c93c7992 +https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.10.1-pyhd8ed1ab_0.conda#1e0d85c0e2fef9539218da185b285f54 https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30c2c15b82aacb07f9c09e28ff2275 -https://conda.anaconda.org/conda-forge/noarch/zipp-3.16.2-pyhd8ed1ab_0.conda#2da0451b54c4563c32490cb1b7cf68a1 +https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda#2e4d6bc0b14e10f895fc6791a7d9b26a https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.4-pyhd8ed1ab_0.conda#46a2e6e3dfa718ce3492018d5a110dd6 https://conda.anaconda.org/conda-forge/noarch/asgiref-3.7.2-pyhd8ed1ab_0.conda#596932155bf88bb6837141550cb721b0 -https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.6-py311h38be061_0.conda#28b1d4a493fb6acd24cc299d77fed871 -https://conda.anaconda.org/conda-forge/noarch/asttokens-2.2.1-pyhd8ed1ab_0.conda#bf7f54dd0f25c3f06ecb82a07341841a -https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.0-hbbaa140_3.conda#cbd8f87157cd414417ade3e9f24274cd -https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.0-h2e270ba_0.conda#bcd553e453b41ce5e4a6154571687b49 -https://conda.anaconda.org/conda-forge/noarch/babel-2.12.1-pyhd8ed1ab_1.conda#ac432e732804a81ddcf29c92ead57cde +https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.8-py311h38be061_0.conda#46d70fcb74472aab178991f0231ee3c6 +https://conda.anaconda.org/conda-forge/noarch/asttokens-2.4.1-pyhd8ed1ab_0.conda#5f25798dcefd8252ce5f9dc494d5f571 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.5-h1a24852_0.conda#02305820d0dbfe542c6e4d67ddb0f13b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.8-h31a96f8_0.conda#cf4834799534b9fcb7bca1c136bcd7a9 +https://conda.anaconda.org/conda-forge/noarch/babel-2.13.1-pyhd8ed1ab_0.conda#3ccff479c246692468f604df9c85ef26 https://conda.anaconda.org/conda-forge/noarch/backports.functools_lru_cache-1.6.5-pyhd8ed1ab_0.conda#6b1b907661838a75d067a22f87996b2e https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.2-pyha770c72_0.conda#a362ff7d976217f8fa78c0f1c4f59717 -https://conda.anaconda.org/conda-forge/noarch/bleach-6.0.0-pyhd8ed1ab_0.conda#d48b143d01385872a88ef8417e96c30e -https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-hbbf8b49_1016.conda#c1dd96500b9b1a75e9e511931f415cbc +https://conda.anaconda.org/conda-forge/noarch/bleach-6.1.0-pyhd8ed1ab_0.conda#0ed9d7c0e9afa7c025807a9a8136ea3e +https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.0-h3faef2a_0.conda#f907bb958910dc404647326ca80c263e https://conda.anaconda.org/conda-forge/noarch/cattrs-23.1.2-pyhd8ed1ab_0.conda#e554f60477143949704bf470f66a81e7 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda#9025d0786dbbe4bc91fd8e85502decce -https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.2.0-hd9d235c_0.conda#8c57a9adbafd87f5eff842abde599cb4 -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h4c7f6c3_1.tar.bz2#c7e54004ffd03f8db0a58ab949f2a00b +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.16.0-py311hb3a22ac_0.conda#b3469563ac5e808b0cd92810d0697043 +https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda#797554b8b7603011e8677884381fbcc5 https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.0-py311h9547e67_0.conda#daf3f23397ab2265d0cdfa339f3627ba -https://conda.anaconda.org/conda-forge/linux-64/coverage-7.2.7-py311h459d7ec_0.conda#3c2c65575c28b23afc5e4ff721a2fc9f -https://conda.anaconda.org/conda-forge/linux-64/curl-8.2.1-hca28451_0.conda#b7bf35457c5495009392c17feec4fddd +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.3.2-py311h459d7ec_0.conda#7b3145fed7adc7c63a0e08f6f29f5480 https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.6.0-h00ab1b0_0.conda#364c6ae36c4e36fcbd4d273cf4db78af -https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_0.conda#5c416db47b7816e437eaf0d46e5c3a3d +https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_1.conda#afe341dbe834ae76d2c23157ff00e633 https://conda.anaconda.org/conda-forge/noarch/docformatter-1.7.5-pyhd8ed1ab_0.conda#3a941b6083e945aa87e739a9b85c82e9 https://conda.anaconda.org/conda-forge/noarch/fire-0.5.0-pyhd8ed1ab_0.conda#9fd22aae8d2f319e80f68b295ab91d64 -https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.42.0-py311h459d7ec_0.conda#8c1ac2c00995248898220c4c1a9d81ab +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.43.1-py311h459d7ec_0.conda#ac995b680de3bdce2531c553b27dfe7e https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.6.0-heb67821_0.conda#b65c49dda97ae497abcbdf3a8ba0018f -https://conda.anaconda.org/conda-forge/noarch/geopy-2.3.0-pyhd8ed1ab_0.tar.bz2#529faeecd6eee3a3b782566ddf05ce92 -https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.10-pyhd8ed1ab_0.conda#3706d2f3d7cb5dae600c833345a76132 -https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.1-nompi_h4f84152_100.conda#ff9ae10aa224826c07da7ef26cb0b717 +https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.0-pyhd8ed1ab_0.conda#90faaa7eaeba3cc877074c0916efe30c +https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.11-pyhd8ed1ab_0.conda#623b19f616f2ca0c261441067e18ae40 +https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.2-nompi_h4f84152_100.conda#2de6a9bc8083b49f09b2f6eb28d3ba3c https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda#4e9f59a060c3be52bc4ddc46ee9b6946 -https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.0.1-pyhd8ed1ab_0.conda#d978c61aa5fc2c69380d53ad56b5ae86 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.1.0-pyhd8ed1ab_0.conda#48b0d98e0c0ec810d3ccc2a0926c8c0e https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 https://conda.anaconda.org/conda-forge/noarch/isort-5.12.0-pyhd8ed1ab_1.conda#07ed3421bad60867234c7a9282ea39d4 -https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.0-pyhd8ed1ab_0.conda#1cd7f70057cdffc10977b613fb75425d +https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.1-pyhd8ed1ab_0.conda#81a3be0b2023e1ea8555781f0ad904a2 https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.2.2-pyhd8ed1ab_0.tar.bz2#243f63592c8e449f40cd42eb5cf32f40 https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 -https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-hfa28ad5_6.conda#ef06bee47510a7f5db3c2297a51d6ce2 -https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h840a212_1.conda#03c225a73835f5aa68c13e62eb360406 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-19_linux64_openblas.conda#d12374af44575413fbbd4a217d46ea33 +https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h119a65a_9.conda#cfebc557e54905dadc355c0e9f003004 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h19a6dae_3.conda#cb26f6b7184480053106ea4713a52daf +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-19_linux64_openblas.conda#9f100edf65436e3eabc2a51fc00b2c37 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.6-pyhd8ed1ab_0.tar.bz2#b21613793fcc81d944c76c9f2864a7de -https://conda.anaconda.org/conda-forge/linux-64/mypy-1.5.0-py311h459d7ec_0.conda#a7c351313fe98e9f26e76b71aa3d41cd +https://conda.anaconda.org/conda-forge/linux-64/mypy-1.6.1-py311h459d7ec_0.conda#d3e5072b7ff746e9f9b44f0f52de6727 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.8.0-pyhd8ed1ab_0.conda#2a75b296096adabbabadd5e9782e5fcc -https://conda.anaconda.org/conda-forge/noarch/partd-1.4.0-pyhd8ed1ab_0.conda#721dab5803ea92ce02ddc4ee50aa0c48 +https://conda.anaconda.org/conda-forge/noarch/partd-1.4.1-pyhd8ed1ab_0.conda#acf4b7c0bcd5fa3b0e05801c4d2accd6 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.8.0-pyh1a96a4e_2.tar.bz2#330448ce4403cc74990ac07c555942a1 -https://conda.anaconda.org/conda-forge/linux-64/pillow-10.0.0-py311h0b84326_0.conda#4b24acdc1fbbae9da03147e7d2cf8c8a -https://conda.anaconda.org/conda-forge/noarch/pip-23.2.1-pyhd8ed1ab_0.conda#e2783aa3f9235225eec92f9081c5b801 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.4-h8972f4a_0.conda#bf6169ef6f83cc04d8b2a72cd5c364bc -https://conda.anaconda.org/conda-forge/linux-64/proj-9.2.1-ha643af7_0.conda#e992387307f4403ba0ec07d009032550 +https://conda.anaconda.org/conda-forge/linux-64/pillow-10.1.0-py311ha6c5da5_0.conda#83a988daf5c49e57f7d2086fb6781fe8 +https://conda.anaconda.org/conda-forge/noarch/pip-23.3.1-pyhd8ed1ab_0.conda#2400c0b86889f43aa52067161e1fb108 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-16.0-h8972f4a_1.conda#6ce1ab5480d3aa4308654971ac5f731b +https://conda.anaconda.org/conda-forge/linux-64/proj-9.3.0-h1d62c97_2.conda#b5e57a0c643da391bef850922963eece https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.3.0-pyhd8ed1ab_0.conda#7e23a61a7fbaedfef6eb0e1ac775c8e5 -https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.0-pyhd8ed1ab_0.conda#3cfe9b9e958e7238a386933c75d190db +https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.3-pyhd8ed1ab_0.conda#5bdca0aca30b0ee62bb84854e027eae0 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 https://conda.anaconda.org/conda-forge/noarch/referencing-0.30.2-pyhd8ed1ab_0.conda#a33161b983172ba6ef69d5fc850650cd -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311h54d622a_1.conda#a894c65b48676c4973e9ee8b59bceb9e https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.2.1-pyhd8ed1ab_0.tar.bz2#7234c9eefff659501cd2fe0d2ede4d48 -https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.2-pyhd8ed1ab_0.conda#700fb06cd011d594305e3b487d5a96a2 -https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.7.1-hd8ed1ab_0.conda#f96688577f1faa58096d06a45136afa2 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.8.0-hd8ed1ab_0.conda#384462e63262a527bda564fa2d9126c0 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 -https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.4-pyhd8ed1ab_0.conda#18badd8fa3648d1beb1fcc7f2e0f756e -https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-h8d71039_2.conda#6d5edbe22b07abae2ea0a9065ef6be12 +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.7-pyhd8ed1ab_0.conda#270e71c14d37074b1d066ee21cf0c4a6 +https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-hac6953d_3.conda#297e6a75dc1b6a440cd341a85eab8a00 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.32.0-pyhd8ed1ab_0.conda#6d2425548b0293a225ca4febd80feaa3 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.13-heb0bb06_2.conda#c0866da05d5e7bb3a3f6b68bcbf7537b -https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_0.conda#43a71a823583d75308eaf3a06c8f150b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.20-he249171_1.conda#e7b72928833ea245d8bfb89a35ae7d5e https://conda.anaconda.org/conda-forge/linux-64/compilers-1.6.0-ha770c72_0.conda#e2259de4640a51a28c21931ae98e4975 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.3-py311h63ff55d_0.conda#cc8ad641cab65dfe59caddbc23a1aeca -https://conda.anaconda.org/conda-forge/noarch/django-4.2.3-pyhd8ed1ab_0.conda#00a369ccfec9deaa325eabd749bfae28 +https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.5-py311h63ff55d_0.conda#22584e5c97ed8f1a6b63a0ff43dba827 +https://conda.anaconda.org/conda-forge/noarch/django-4.2.7-pyhd8ed1ab_0.conda#107fbbd296283091c717acd51a3a87fd https://conda.anaconda.org/conda-forge/noarch/flake8-5.0.4-pyhd8ed1ab_0.tar.bz2#8079ea7dec0a917dd0cb6c257f7ea9ea -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h22adcc9_11.conda#514167b60f598eaed3f7a60e1dceb9ee -https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.32-pyhd8ed1ab_0.conda#5809a12901d57388444c3293c975d0bb -https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-7.3.0-hdb3a94d_0.conda#765bc76c0dfaf24ff9d8a2935b2510df +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-hf074850_14.conda#1d53ee057d8481bd2b4c2c34c8e92aac +https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.40-pyhd8ed1ab_0.conda#6bf74c3b7c13079a91d4bd3da51cefcf +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.2.1-h3d44ed6_0.conda#98db5f8813f45e2b29766aff0e4a499c https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda#b279b07ce18058034e5b3606ba103a8b https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.7.1-pyhd8ed1ab_0.conda#7c27ea1bdbe520bb830dcadd59f55cbf -https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-h3e6883b_4.conda#ef5228158594262a8bc07a0c92a3ef5b -https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-hca56755_27.conda#918a735059cab21b96fc13a8d04fbcd8 -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.7.2-py311h54ef318_0.conda#2631a9e423855fb586c05f8a5ee8b177 -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.0.3-py311h320fe9a_1.conda#5f92f46bd33917832a99d1660b4075ac -https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda#0809187ef9b89a3d94a5c24d13936236 -https://conda.anaconda.org/conda-forge/linux-64/poppler-23.05.0-hd18248d_1.conda#09e0de1aa7330fe697eed76eaeef666d +https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.2-hcd42e92_1.conda#b04c039f0bd511533a0d8bc8a7b6835e +https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h80fb2b6_112.conda#a19fa6cacf80c8a366572853d5890eb4 +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h090f1da_0.conda#c4360eaa543bb3bcbb9cd135eb6fb0fc +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.0-py311h64a7726_0.conda#bf16a9f625126e378302f08e7ed67517 +https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.11.0-pyhd8ed1ab_0.conda#8f567c0a74aa44cf732f15773b4083b0 +https://conda.anaconda.org/conda-forge/linux-64/poppler-23.10.0-h590f24d_0.conda#06b4a80e2cc3974e55f83e2115e2e90a https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.0-py311ha169711_1.conda#92633556d37e88ce45193374d408072c +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311h1facc83_3.conda#fee8c3c4404e37e724f08c2983eb484c https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.1.0-pyhd8ed1ab_0.conda#06eb685a3a0b146347a58dda979485da -https://conda.anaconda.org/conda-forge/noarch/pytest-env-0.8.2-pyhd8ed1ab_0.conda#cd20b56ff31042aad74d45e3f47eb5b2 +https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.1.1-pyhd8ed1ab_0.conda#0a4d7e888fae7b7d75243414b15d0c50 https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.0.0-pyhd8ed1ab_1.conda#8bdcc0f401561213821bf67513abeeff -https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.11.1-pyhd8ed1ab_0.conda#fcd2531bc3e492657aeb042349aeaf8a +https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.12.0-pyhd8ed1ab_0.conda#ac9fedc9a0c397f2318e82525491dd83 https://conda.anaconda.org/conda-forge/noarch/pytest-mypy-0.8.0-pyhd8ed1ab_0.tar.bz2#4e81c96e5f875c09e5b9f999035b9d8e https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.3.1-pyhd8ed1ab_0.conda#816073bb54ef59f33f0f26c14f88311b https://conda.anaconda.org/conda-forge/noarch/rdflib-7.0.0-pyhd8ed1ab_0.conda#44d14ef95495b3d4438f28998e0296a9 https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda#a30144e4156cdbb236f99ebb49828f8b https://conda.anaconda.org/conda-forge/noarch/requirements-detector-1.2.2-pyhd8ed1ab_0.conda#6626918380d99292df110f3c91b6e5ec https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.2-pyhd8ed1ab_0.conda#e7df0fdd404616638df5ece6e69ba7af -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.13.2-hd532e3d_0.conda#6d97164f19dbd27575ef1899b02dc1e0 -https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h4dd048b_3.tar.bz2#dbfea4376856bf7bd2121e719cf816e5 -https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.6-pyhd8ed1ab_0.conda#078979d33523cb477bd1916ce41aacc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.21.0-h87b6960_2.conda#daacb517f0c54d561a1c8f793cc24406 -https://conda.anaconda.org/conda-forge/noarch/bokeh-3.2.1-pyhd8ed1ab_0.conda#1f6a50747c4e69d181f3a2ca739d1fe3 -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_0.conda#1271b2375735e2aaa6d6770dbe2ad087 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.8.0-pyhd8ed1ab_0.conda#160a92928fc4a0ca40a64b586a2cf671 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h8c794c1_3.conda#7de728789b0aba16018f726dc5ddbec2 +https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.10-pyhd8ed1ab_0.conda#84dca7318667fa091bbfd8724d22ea99 +https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h9547e67_4.conda#586da7df03b68640de14dc3e8bcbf76f +https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.9-pyhd8ed1ab_0.conda#8e8280dec091763dfdc29e066de52270 +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.24.4-h28e6ea9_2.conda#12fcf00d22c16ae8663d2f0fe0068d0e +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.3-py311h1f0f07a_0.conda#b7e6d52b39e199238c3400cafaabafb3 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.1-py311h9547e67_1.conda#52d3de443952d33c5cee6b24b172ce96 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.10.1-pyhd8ed1ab_0.conda#cf1f26a9e21311f10cfe9dbf0e0d99bc https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe -https://conda.anaconda.org/conda-forge/noarch/identify-2.5.26-pyhd8ed1ab_0.conda#1ca86f154e13f4aa20b48e20d6bbf924 -https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.0-pyhd8ed1ab_0.conda#989d1df249e1fce06fa1435548eef68a -https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.3.1-py311h38be061_0.conda#0cf8259b01ede82c76007996f73f89ed -https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 -https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-heaa33ce_1.conda#cde553e0e32389e26595db4eacf859eb -https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda#5936894aade8240c867d292aa0d980c6 +https://conda.anaconda.org/conda-forge/noarch/identify-2.5.31-pyhd8ed1ab_0.conda#fea10604a45e974b110ea15a88913ebc +https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.2-pyhd8ed1ab_0.conda#24d41c2f9cc199d0a180ecf7ef54739c +https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.5.0-py311h38be061_0.conda#cee83be29258275f75029125e186ab6d +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.2-h6f3d308_7.conda#f9555eada0412c9f3dd4b34f5afecf5b +https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hacb5139_102.conda#487a1c19dd3eacfd055ad614e9acde87 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.2-py311h320fe9a_0.conda#c36a53056129665b34db419b6af3d230 +https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-ha41ecd1_2.conda#1a66c10f6a0da3dbd2f3a68127e7f6a0 +https://conda.anaconda.org/conda-forge/noarch/pooch-1.8.0-pyhd8ed1ab_0.conda#134b2b57b7865d2316a7cce1915a51ed https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.39-pyha770c72_0.conda#a4986c6bb5b0d05a38855b0880a5f425 -https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.5-pyhd8ed1ab_0.conda#30dc94b05de470e3b579d73d64127656 -https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.2.0-pyhd8ed1ab_1.conda#34f7d568bf59d18e3fef8c405cbece21 +https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.7-pyhd8ed1ab_0.conda#3cab6aee60038b3f621bce3e50f52bed +https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.3.0-pyhd8ed1ab_0.conda#7819533e674dbbc51468f3228b9b1bb6 https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.1.0-pyhd8ed1ab_0.conda#57b89064c125bb9d0e533e018c3eb17a -https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.2-pyhd8ed1ab_0.conda#a218f3be8ab6185a475c8168a86e18ae -https://conda.anaconda.org/conda-forge/noarch/xarray-2023.7.0-pyhd8ed1ab_0.conda#2f18700699e1ea19aa1634ed57711677 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.3-py311h64a7726_1.conda#e4b4d3b764e2d029477d0db88248a8b5 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.2-py311he06c224_0.conda#c90e2469d7512f3bba893533a82d7a02 +https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.6-pyhd8ed1ab_0.conda#fb1fc875719e217ed799a7aae11d3be4 https://conda.anaconda.org/conda-forge/noarch/yapf-0.40.1-pyhd8ed1ab_0.conda#f269942e802d5e148632143d4c37acc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.10.57-h7062fed_18.conda#695364bcdf9336c2108382c48a6efa97 -https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 -https://conda.anaconda.org/conda-forge/noarch/distributed-2023.8.0-pyhd8ed1ab_0.conda#974b4a00b0e100e341cd9f179b05f574 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.182-hb97d603_2.conda#4b28dbada9459f1d89c77939b9284388 +https://conda.anaconda.org/conda-forge/noarch/bokeh-3.3.0-pyhd8ed1ab_0.conda#5d6ff9d18f0b611a7dc131f4a7444c2e +https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_4.conda#1e105c1a8ea2163507726144b401eb1b +https://conda.anaconda.org/conda-forge/noarch/distributed-2023.10.1-pyhd8ed1ab_0.conda#9fc9e3eacf6e23f902408d38198880de +https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_h9e768e6_3.conda#c330e87e698bae8e7381c0315cf25dd0 +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.2-py311h815a124_7.conda#e68c90c9490e1621ff4a7871849f03b2 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 -https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.3.0-pyhd8ed1ab_0.conda#1d018ee4ab13217e2544f795eb0a6798 +https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.5.0-pyhd8ed1ab_0.conda#77e442cb7c382d01c916f91a8652811a https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda#620e754f4344f4c27259ff460a2b9c50 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.1-py311h54ef318_0.conda#201fdabdb86bb8fb6e99fa3f0dab8122 https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.2-pyhd8ed1ab_0.conda#61ba076de6530d9301a0053b02f093d2 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.5-nompi_py311he8ad708_100.conda#597b1ad6cb7011b7561c20ea30295cae https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d -https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.3.3-pyha770c72_0.conda#dd64a0e440754ed97610b3e6b502b6b1 +https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.5.0-pyha770c72_0.conda#964e3d762e427661c59263435a14c492 https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.39-hd8ed1ab_0.conda#4bbbe67d5df19db30f04b8e344dc9976 https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 -https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_0.conda#3a00b1b08d8c01b1a3bfa686b9152df2 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.1-py311h64a7726_0.conda#356da36102fc1eeb8a81e6d79e53bc7e +https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_1.conda#cd36a89a048ad2bcc6d8b43f648fb1d0 +https://conda.anaconda.org/conda-forge/noarch/xarray-2023.10.1-pyhd8ed1ab_0.conda#9b20e5d68eea6878a0a6fc57a3043889 +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_1.conda#10d1806e20da040c58c36deddf51c70c +https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.8.2-pyhd8ed1ab_0.conda#cc344a296a41369bcb05f7216661cec8 https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyh1a96a4e_2.tar.bz2#64068564a9c2932bf30e9b4ec567927d +https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda#ddcf387719b2e44df0cc4dd467643951 +https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.5-py311hbac4ec9_0.conda#786d3808394b1bdfd3f41f2e2c67279e https://conda.anaconda.org/conda-forge/linux-64/graphviz-8.1.0-h28d9a01_0.conda#33628e0e3de7afd2c8172f76439894cb -https://conda.anaconda.org/conda-forge/noarch/ipython-8.14.0-pyh41d4057_0.conda#0a0b0d8177c4a209017b356439292db8 -https://conda.anaconda.org/conda-forge/linux-64/libarrow-12.0.1-h10ac928_8_cpu.conda#5f3fef49968b171fda2c8bdf2adf7051 -https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h7e745eb_109.conda#9e208615247477427acbd0900ca7038f +https://conda.anaconda.org/conda-forge/noarch/ipython-8.17.2-pyh41d4057_0.conda#f39d0b60e268fe547f1367edbab457d4 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-hecbb4c5_13_cpu.conda#71172fd3f165406793843c3211248169 https://conda.anaconda.org/conda-forge/noarch/nbclient-0.8.0-pyhd8ed1ab_0.conda#e78da91cf428faaf05701ce8cc8f2f9b -https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.2-pyhd8ed1ab_0.conda#457fae5edf0703054b78f6bfb200e855 +https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.0-h4a547c6_3.conda#a41bb5d9bbb3c80c041fbdef33bd27d5 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.7.3-pyhd8ed1ab_0.conda#063c1fda5480050b8d989478c97a4c55 -https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hec59055_101.conda#c84dbed01258db73689f72abc01c5e1a -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311h9a7c333_101.conda#1dc70c7c3352c0ff1f861d866860db37 -https://conda.anaconda.org/conda-forge/noarch/prospector-1.10.2-pyhd8ed1ab_0.conda#2c536985982f7e531df8d640f554008a -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-12.0.1-py311h39c9aba_8_cpu.conda#587370a25bb2c50cce90909ce20d38b8 -https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_3.tar.bz2#64a77de29fde80aef5013ddf5e62a564 -https://conda.anaconda.org/conda-forge/noarch/dask-2023.8.0-pyhd8ed1ab_0.conda#0cd5f8a91edc681f4b87e8a0ce33a591 -https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_ha7f9e30_1.conda#f3516df9a5e2b2ef3e3be2b350f9e93d -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.0-py311h1caf18b_3.conda#7502ec69aa41329bd65aa44eb2853738 -https://conda.anaconda.org/conda-forge/noarch/iris-3.6.1-pyha770c72_0.conda#36d615b339058273520990fc78239116 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.7.3-pyhd8ed1ab_0.conda#f44109e52a40b8149156f5ddd9c11b26 +https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc1f660bf455c239adaabf56b91dc9 +https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.10.0-pyhd8ed1ab_0.conda#56c421c936a17bbc7fd4b36c9bd3b236 +https://conda.anaconda.org/conda-forge/noarch/prospector-1.10.3-pyhd8ed1ab_0.conda#f551d4d859a1d70c6abff8310a655481 +https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.5-pyhd8ed1ab_0.conda#28c3e17587c80297c4d9b1bc4cbba917 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_13_cpu.conda#9b9c895aa2414d8c27e2e7818d13bef7 +https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_4.conda#5c223cb0d9c05552bf9d1586a92720b2 +https://conda.anaconda.org/conda-forge/noarch/dask-2023.10.1-pyhd8ed1ab_0.conda#5b5af2efa7659441a9967734d046fecf +https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.10.0-pyhd8ed1ab_0.conda#2307d9331078c3097631ac2129d4f3cf https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 -https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_1.conda#4067029ad6872d49f6d43c05dd1f51a9 -https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.4-py311hbac4ec9_0.conda#1d3445f5f7fa002a1c149c405376f012 -https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.7.3-pyhd8ed1ab_0.conda#f53d92ecd7d8563b006107f6a33e55c6 -https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.7.0-pyhd8ed1ab_0.conda#de82eb8d09362babacafe6b7e27752ac +https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.8.0-pyhd8ed1ab_0.conda#56e85460d22fa7d4fb06300f785dd1e1 +https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.10.0-pyhd8ed1ab_0.conda#800568b9c39cfd87a2a6a9a0488ea265 https://conda.anaconda.org/conda-forge/noarch/autodocsumm-0.2.6-pyhd8ed1ab_0.tar.bz2#4409dd7e06a62c3b2aa9e96782c49c6d -https://conda.anaconda.org/conda-forge/noarch/nbsphinx-0.9.2-pyhd8ed1ab_0.conda#d1212b423fdd10d2da59601385561ff7 -https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.13.3-pyhd8ed1ab_0.conda#07aca5f2dea315dcc16680d6891e9056 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.6-pyhd8ed1ab_0.conda#5bba7b5823474cb3fcd4e4cbf942da61 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.4-pyhd8ed1ab_0.conda#73dcd0eb2252cbd1530fd1e6e3cbbb03 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.3-pyhd8ed1ab_0.conda#fb4d6329a57e20e03d7aecd18c7ca918 -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.5-pyhd8ed1ab_0.conda#85466265b76473cc1d02420056cbc4e3 -https://conda.anaconda.org/conda-forge/noarch/sphinx-7.1.2-pyhd8ed1ab_0.conda#d02bfa35cd4f2cd624289f64911cae9d -https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.7-pyhd8ed1ab_0.conda#01e35beea8aff61cdb445b90a7adf7d4 +https://conda.anaconda.org/conda-forge/noarch/nbsphinx-0.9.3-pyhd8ed1ab_0.conda#0dbaa7d08d3d79b2a1a4dd6a02cc4581 +https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.14.3-pyhd8ed1ab_0.conda#3a52e29fde69705040bcbe74cbdcbd5c +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.7-pyhd8ed1ab_0.conda#aebfabcb60c33a89c1f9290cab49bc93 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.5-pyhd8ed1ab_0.conda#ebf08f5184d8eaa486697bc060031953 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.4-pyhd8ed1ab_0.conda#a9a89000dfd19656ad004b937eeb6828 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.6-pyhd8ed1ab_0.conda#cf5c9649272c677a964a7313279e3a9b +https://conda.anaconda.org/conda-forge/noarch/sphinx-7.2.6-pyhd8ed1ab_0.conda#bbfd1120d1824d2d073bc65935f0e4c0 +https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.9-pyhd8ed1ab_0.conda#0612e497d7860728f2cda421ea2aec09 diff --git a/doc/changelog.rst b/doc/changelog.rst index b8718686c7..daa5dae664 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -3,6 +3,144 @@ Changelog ========= +.. _changelog-v2-10-0: + +v2.10.0 +------- +Highlights + +TODO: add highlights + +This release includes + +Backwards incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove the deprecated option ``use_legacy_supplementaries`` (`#2202 `__) `Bouwe Andela `__ + + - The recommended upgrade procedure is to remove ``use_legacy_supplementaries`` from config-user.yml + (if it was there) and remove any mention of ``fx_variables`` from the recipe. If automatically defining + the required supplementary variables does not work, define them in the variable or + (``additional_``) ``datasets`` section as described in :ref:`supplementary_variables`. + +- Use smarter (units-aware) weights (`#2139 `__) `Manuel Schlund `__ + + - Some preprocessors handle units better. For details, see the pull request. + +- Removed deprecated configuration option ``offline`` (`#2213 `__) `Manuel Schlund `__ + + - In :ref:`changelog-v2-8-0`, we replaced the old ``offline`` configuration option. From this version on, it stops working. + Please refer to :ref:`changelog-v2-8-0` for upgrade instructions. + +- Fix issue with CORDEX datasets requiring different dataset tags for downloads and fixes (`#2066 `__) `Joakim Löw `__ + + - Due to the different facets for CORDEX datasets, there was an inconsistency in the fixing mechanism. + This change requires changes to existing recipes that use CORDEX datasets. Please refer to the pull request for detailed update instructions. + +- Added new operators for statistics preprocessor (e.g., ``'percentile'``) and allowed passing additional arguments (`#2191 `__) `Manuel Schlund `__ + + - This harmonizes the operators for all statistics preprocessors. From this version, the new names can be used; the old arguments will stop working from + version 2.12.0. Please refer to :ref:`stat_preprocs` for a detailed description. + +- For the following changes, no user change is necessary + + - Remove deprecated way of calling :func:`~esmvalcore.cmor.table.read_cmor_tables` (`#2201 `__) `Bouwe Andela `__ + + - Remove deprecated callback argument from preprocessor ``load`` function (`#2207 `__) `Bouwe Andela `__ + + - Remove deprecated preprocessor function `cleanup` (`#2215 `__) `Bouwe Andela `__ + + +Deprecations +~~~~~~~~~~~~ + +- Clearly separate fixes and CMOR checks (`#2157 `__) `Manuel Schlund `__ + +Bug fixes +~~~~~~~~~ + +- Re-add correctly region-extracted cell measures and ancillary variables after :ref:`extract_region` (`#2166 `__) `Valeriu Predoi `__, `Manuel Schlund `__ +- Fix sorting of datasets + + - Fix sorting of ensemble members in :func:`~esmvalcore.dataset.datasets_to_recipe` (`#2095 `__) `Bouwe Andela `__ + - Fix a problem with sorting datasets that have a mix of facet types (`#2238 `__) `Bouwe Andela `__ + - Avoid a crash if dataset has supplementary variables (`#2198 `__) `Bouwe Andela `__ + +CMOR standard +~~~~~~~~~~~~~ + +- ERA5 on-the-fly CMORizer: changed sign of variables ``evspsbl`` and ``evspsblpot`` (`#2115 `__) `katjaweigel `__ +- Add ``ch4`` surface custom cmor table entry (`#2168 `__) `Birgit Hassler `__ +- Add CMIP3 institutes names used at NCI (`#2152 `__) `Romain Beucher `__ +- Added :func:`~esmvalcore.cmor.fixes.get_time_bounds` and :func:`~esmvalcore.cmor.fixes.get_next_month` to public API (`#2214 `__) `Manuel Schlund `__ +- Improve concatenation checks + + - Relax concatenation checks for ``--check_level=relax`` and ``--check_level=ignore`` (`#2144 `__) `sloosvel `__ + - Fix ``concatenate`` preprocessor function (`#2240 `__) `Bouwe Andela `__ + - Fix time overlap handling in concatenation (`#2247 `__) `Klaus Zimmermann `__ + +Computational performance improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Make :ref:`threshold_masking` preprocessors lazy (`#2169 `__) `Jörg Benke `__ + + - Restored usage of numpy in `_mask_with_shp` (`#2209 `__) `Jörg Benke `__ +- Call coord.core_bounds() instead of coord.bounds in ``check.py`` (`#2146 `__) `sloosvel `__ +- Rechunk between preprocessor steps (`#2205 `__) `Bouwe Andela `__ +- Reduce the size of the dask graph created by the ``anomalies`` preprocessor function (`#2200 `__) `Bouwe Andela `__ + +Documentation +~~~~~~~~~~~~~ + +- Add reference to release v2.9.0 in the changelog (`#2130 `__) `Rémi Kazeroni `__ +- Add merge instructions to release instructions (`#2131 `__) `Klaus Zimmermann `__ +- Update `mamba` before building environment during Readthedocs build (`#2149 `__) `Valeriu Predoi `__ +- Ensure compatible zstandard and zstd versions for .conda support (`#2204 `__) `Klaus Zimmermann `__ +- Remove outdated documentation (`#2210 `__) `Bouwe Andela `__ +- Remove meercode badge from README because their API is broken (`#2224 `__) `Valeriu Predoi `__ +- Correct usage help text of version command (`#2232 `__) `James Frost `__ +- Add ``navigation_with_keys: False`` to ``html_theme_options`` in Readthedocs ``conf.py`` (`#2245 `__) `Valeriu Predoi `__ +- Replace squarey badge with roundy shield for Anaconda sticker in README (`#2233 `__) `Valeriu Predoi `__ + +Fixes for datasets +~~~~~~~~~~~~~~~~~~ + +- Updated doc about fixes and added type hints to fix functions (`#2160 `__) `Manuel Schlund `__ + +Installation +~~~~~~~~~~~~ + +- Clean-up how pins are written in conda environment file (`#2125 `__) `Valeriu Predoi `__ +- Use importlib.metadata instead of deprecated pkg_resources (`#2096 `__) `Bouwe Andela `__ +- Pin shapely to >=2.0 (`#2075 `__) `Valeriu Predoi `__ + +Preprocessor +~~~~~~~~~~~~ + +- Improve preprocessor output sorting code (`#2111 `__) `Bouwe Andela `__ +- Preprocess datasets in the same order as they are listed in the recipe (`#2103 `__) `Bouwe Andela `__ + +Automatic testing +~~~~~~~~~~~~~~~~~ + +- [Github Actions] Compress all bash shell setters into one default option per workflow (`#2126 `__) `Valeriu Predoi `__ +- [Github Actions] Fix Monitor Tests Github Action (`#2135 `__) `Valeriu Predoi `__ +- [condalock] update conda lock file (`#2141 `__) `Valeriu Predoi `__ +- [Condalock] make sure mamba/conda are at latest version by forcing a pinned mamba install (`#2136 `__) `Valeriu Predoi `__ +- Update code coverage orbs (`#2206 `__) `Bouwe Andela `__ +- Revisit the comment-triggered Github Actions test (`#2243 `__) `Valeriu Predoi `__ +- Remove workflow that runs Github Actions tests from PR comment (`#2244 `__) `Valeriu Predoi `__ + +Improvements +~~~~~~~~~~~~ + +- Merge v2.9.x into main (`#2128 `__) `Manuel Schlund `__ +- Fix typo in citation file (`#2182 `__) `Bouwe Andela `__ +- Cleaned and extended function that extracts datetimes from paths (`#2181 `__) `Manuel Schlund `__ +- Add file encoding (and some read modes) at open file step (`#2219 `__) `Valeriu Predoi `__ +- Check type of argument passed to :func:`~esmvalcore.cmor.table.read_cmor_tables` (`#2217 `__) `Valeriu Predoi `__ +- Dynamic HTML output for monitoring (`#2062 `__) `Brei Soliño `__ + .. _changelog-v2-9-0: diff --git a/doc/conf.py b/doc/conf.py index a71f331378..1613a0133b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -152,7 +152,19 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -# html_theme_options = {} +# +# Avoid the following warning issued by pydata_sphinx_theme: +# +# "WARNING: The default value for `navigation_with_keys` will change to `False` +# in the next release. If you wish to preserve the old behavior for your site, +# set `navigation_with_keys=True` in the `html_theme_options` dict in your +# `conf.py` file.Be aware that `navigation_with_keys = True` has negative +# accessibility implications: +# https://github.com/pydata/pydata-sphinx-theme/issues/1492" +# Short synopsis of said issue: as of now, left/right keys take one +# to the previous/next page instead of scrolling horizontally; this +# should be fixed upstream, then we can set again navigation with keys True +html_theme_options = {"navigation_with_keys": False} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] diff --git a/doc/contributing.rst b/doc/contributing.rst index 8201fac707..814ab79263 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -550,18 +550,6 @@ and the result of the tests ran by GitHub Actions can be viewed on the of the repository (to learn more about the Github-hosted runners, please have a look the `documentation `__). -When opening a pull request, if you wish to run the Github Actions `Test `__ test, -you can activate it via a simple comment containing the @runGAtests tag -(e.g. "@runGAtests" or "@runGAtests please run" - in effect, tagging the runGAtests -bot that will start the test automatically). This is useful -to check if a certain feature that you included in the Pull Request, and can be tested -for via the test suite, works across the supported Python versions, and both on Linux and OSX. -The test is currently deactivated, so before triggering the test via comment, make sure you activate -the test in the main `Actions page `__ -(click on Test via PR Comment and activate it); also and be sure to deactivate it afterwards -(the Github API still needs a bit more development, and currently it triggers -the test for **each comment** irrespective of PR, that's why this needs to be activated/decativated). - The configuration of the tests run by CircleCI can be found in the directory `.circleci `__, while the configuration of the tests run by GitHub Actions can be found in the diff --git a/doc/gensidebar.py b/doc/gensidebar.py index 970722ff0a..01f8b3e839 100644 --- a/doc/gensidebar.py +++ b/doc/gensidebar.py @@ -10,13 +10,13 @@ def _write_if_changed(fname, contents): """Write/update file only if changed.""" try: - with open(fname, "r") as stream: + with open(fname, "r", encoding="utf-8") as stream: old_contents = stream.read() except IOError: old_contents = "" if old_contents != contents: - with open(fname, "w") as stream: + with open(fname, "w", encoding="utf-8") as stream: stream.write(contents) diff --git a/doc/quickstart/run.rst b/doc/quickstart/run.rst index 03eec25444..ebde6d4075 100644 --- a/doc/quickstart/run.rst +++ b/doc/quickstart/run.rst @@ -78,7 +78,9 @@ or This feature is available for projects that are hosted on the ESGF, i.e. CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs. -To control the strictness of the CMOR checker, use the flag ``--check_level``: +To control the strictness of the CMOR checker and the checks during concatenation +on auxiliary coordinates, supplementary variables, and derived coordinates, +use the flag ``--check_level``: .. code:: bash @@ -86,10 +88,10 @@ To control the strictness of the CMOR checker, use the flag ``--check_level``: Possible values are: - - `ignore`: all errors will be reported as warnings - - `relaxed`: only fail if there are critical errors - - `default`: fail if there are any errors - - `strict`: fail if there are any warnings + - `ignore`: all errors will be reported as warnings. Concatenation will be performed without checks. + - `relaxed`: only fail if there are critical errors. Concatenation will be performed without checks. + - `default`: fail if there are any errors. + - `strict`: fail if there are any warnings. To re-use pre-processed files from a previous run of the same recipe, you can use diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 8a0c1a54b7..5c4b14e351 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -7,6 +7,8 @@ Preprocessor In this section, each of the preprocessor modules is described, roughly following the default order in which preprocessor functions are applied: +* :ref:`Overview` +* :ref:`stat_preprocs` * :ref:`Variable derivation` * :ref:`CMOR check and dataset-specific fixes` * :ref:`preprocessors_using_supplementary_variables` @@ -30,6 +32,8 @@ roughly following the default order in which preprocessor functions are applied: See :ref:`preprocessor_functions` for implementation details and the exact default order. +.. _overview: + Overview ======== @@ -52,6 +56,136 @@ for instance, for multi-model statistics, which required the model to be on a common grid and therefore has to be called after the regridding module. +.. _stat_preprocs: + +Statistical preprocessors +========================= + +Many preprocessors calculate statistics over data. +Those preprocessors typically end with ``_statistics``, e.g., +:func:`~esmvalcore.preprocessor.area_statistics` or +:func:`~esmvalcore.preprocessor.multi_model_statistics`. +All these preprocessors support the options `operator`, which directly +correspond to :class:`iris.analysis.Aggregator` objects used to perform the +statistical calculations. +In addition, arbitrary keyword arguments can be passed which are directly +passed to the corresponding :class:`iris.analysis.Aggregator` object. + +.. note:: + The preprocessors :func:`~esmvalcore.preprocessor.multi_model_statistics` + and :func:`~esmvalcore.preprocessor.ensemble_statistics` support the + computation of multiple statistics at the same time. + In these cases, they are defined by the option `statistics` (instead of + `operator`), which takes a list of possible operators. + Each operator can be given as single string or as dictionary. + In the latter case, the dictionary needs the keyword `operator` + (corresponding to the `operator` as above). + All other keywords are interpreted as keyword arguments for the given + operator. + +Some operators support weights for some preprocessors (see following table), +which are used by default. +The following operators are currently fully supported; other operators might be +supported too if proper keyword arguments are specified: + +.. _supported_stat_operator: + +============================== ================================================= ===================================== +`operator` Corresponding :class:`~iris.analysis.Aggregator` Weighted? [1]_ +============================== ================================================= ===================================== +``gmean`` :const:`iris.analysis.GMEAN` no +``hmean`` :const:`iris.analysis.HMEAN` no +``max`` :const:`iris.analysis.MAX` no +``mean`` :const:`iris.analysis.MEAN` yes +``median`` :const:`iris.analysis.MEDIAN` [2]_ no +``min`` :const:`iris.analysis.MIN` no +``peak`` :const:`iris.analysis.PEAK` no +``percentile`` :const:`iris.analysis.PERCENTILE` no +``rms`` :const:`iris.analysis.RMS` yes +``std_dev`` :const:`iris.analysis.STD_DEV` no +``sum`` :const:`iris.analysis.SUM` yes +``variance`` :const:`iris.analysis.VARIANCE` no +``wpercentile`` :const:`iris.analysis.WPERCENTILE` yes +============================== ================================================= ===================================== + +.. [1] The following preprocessor support weighted statistics by default: + :func:`~esmvalcore.preprocessor.area_statistics`: weighted by grid cell + areas (see also :ref:`preprocessors_using_supplementary_variables`); + :func:`~esmvalcore.preprocessor.climate_statistics`: weighted by lengths of + time intervals; :func:`~esmvalcore.preprocessor.volume_statistics`: + weighted by grid cell volumes (see also + :ref:`preprocessors_using_supplementary_variables`); + :func:`~esmvalcore.preprocessor.axis_statistics`: weighted by + corresponding coordinate bounds. +.. [2] :const:`iris.analysis.MEDIAN` is not lazy, but much faster than + :const:`iris.analysis.PERCENTILE`. For a lazy median, use ``percentile`` + with the keyword argument ``percent: 50``. + +Examples +-------- + +Calculate the global (weighted) mean: + +.. code-block:: yaml + + preprocessors: + global_mean: + area_statistics: + operator: mean + +Calculate zonal maximum. + +.. code-block:: yaml + + preprocessors: + zonal_max: + zonal_statistics: + operator: max + +Calculate the 95% percentile over each month separately (will result in 12 time +steps, one for January, one for February, etc.): + +.. code-block:: yaml + + preprocessors: + monthly_percentiles: + climate_statistics: + period: monthly + operator: percentile + percent: 95.0 + +Calculate multi-model median, 5%, and 95% percentiles: + +.. code-block:: yaml + + preprocessors: + mm_stats: + multi_model_statistics: + span: overlap + statistics: + - operator: percentile + percent: 5 + - operator: median + - operator: percentile + percent: 95 + +Calculate the global non-weighted root mean square: + +.. code-block:: yaml + + preprocessors: + global_mean: + area_statistics: + operator: rms + weighted: false + +.. warning:: + + The disabling of weights by specifying the keyword argument ``weights: + False`` needs to be used with great care; from a scientific standpoint, we + strongly recommend to **not** use it! + + .. _Variable derivation: Variable derivation @@ -300,134 +434,6 @@ and cell measure (``areacella``), but do not use ``areacella`` for dataset timerange: '1990/2000' scripts: null - -.. _`Fx variables as cell measures or ancillary variables`: - -Legacy method of specifying supplementary variables ---------------------------------------------------- - -.. deprecated:: 2.8.0 - The legacy method of specifying supplementary variables is deprecated and will - be removed in version 2.10.0. - To upgrade, remove all occurrences of ``fx_variables`` from your recipes and - rely on automatically defining the supplementary variables based on the - requirement of the preprocessor functions or specify them using the methods - described above. - To keep using the legacy behaviour until v2.10.0, set - ``use_legacy_supplementaries: true`` in the :ref:`user configuration file` or - run the tool with the flag ``--use-legacy-supplementaries=True``. - -Prior to version 2.8.0 of the tool, the supplementary variables could not be -defined at the variable or dataset level in the recipe, but could only be -defined in the preprocessor function that uses them using the ``fx_variables`` -argument. -This does not work well because in practice different datasets store their -supplementary variables under different facets. -For example, one dataset might only provide the ``areacella`` variable under the -``1pctCO2`` experiment while another one might only provide it for the -``historical`` experiment. -This forced the user to define a preprocessor per dataset, which was -inconvenient. - -============================================================== ===================== -Preprocessor Default fx variables -============================================================== ===================== -:ref:`area_statistics` ``areacella``, ``areacello`` -:ref:`mask_landsea` ``sftlf``, ``sftof`` -:ref:`mask_landseaice` ``sftgif`` -:ref:`volume_statistics` ``volcello`` -:ref:`weighting_landsea_fraction` ``sftlf``, ``sftof`` -============================================================== ===================== - -If the option ``fx_variables`` is not explicitly specified for these -preprocessors, the default fx variables in the second column are automatically -used. If given, the ``fx_variables`` argument specifies the fx variables that -the user wishes to input to the corresponding preprocessor function. The user -may specify these by simply adding the names of the variables, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - volcello: - -or by additionally specifying further keys that are used to define the fx -datasets, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - mip: Ofx - exp: piControl - volcello: - mip: Omon - -This might be useful to select fx files from a specific ``mip`` table or from a -specific ``exp`` in case not all experiments provide the fx variable. - -Alternatively, the ``fx_variables`` argument can also be specified as a list: - -.. code-block:: yaml - - fx_variables: ['areacello', 'volcello'] - -or as a list of dictionaries: - -.. code-block:: yaml - - fx_variables: [{'short_name': 'areacello', 'mip': 'Ofx', 'exp': 'piControl'}, {'short_name': 'volcello', 'mip': 'Omon'}] - -The recipe parser will automatically find the data files that are associated -with these variables and pass them to the function for loading and processing. - -If ``mip`` is not given, ESMValCore will search for the fx variable in all -available tables of the specified project. - -.. warning:: - Some fx variables exist in more than one table (e.g., ``volcello`` exists in - CMIP6's ``Odec``, ``Ofx``, ``Omon``, and ``Oyr`` tables; ``sftgif`` exists - in CMIP6's ``fx``, ``IyrAnt`` and ``IyrGre``, and ``LImon`` tables). If (for - a given dataset) fx files are found in more than one table, ``mip`` needs to - be specified, otherwise an error is raised. - -.. note:: - To explicitly **not** use any fx variables in a preprocessor, use - ``fx_variables: null``. While some of the preprocessors mentioned above do - work without fx variables (e.g., ``area_statistics`` or ``mask_landsea`` - with datasets that have regular latitude/longitude grids), using this option - is **not** recommended. - -Internally, the required ``fx_variables`` are automatically loaded by the -preprocessor step ``add_fx_variables`` which also checks them against CMOR -standards and adds them either as ``cell_measure`` (see `CF conventions on cell -measures -`_ -and :class:`iris.coords.CellMeasure`) or ``ancillary_variable`` (see `CF -conventions on ancillary variables -`_ -and :class:`iris.coords.AncillaryVariable`) inside the cube data. This ensures -that the defined preprocessor chain is applied to both ``variables`` and -``fx_variables``. - -Note that when calling steps that require ``fx_variables`` inside diagnostic -scripts, the variables are expected to contain the required ``cell_measures`` or -``Fx variables as cell measures or ancillary variables``. If missing, they can be added using the following functions: - -.. code-block:: - - from esmvalcore.preprocessor import (add_cell_measure, add_ancillary_variable) - - cube_with_area_measure = add_cell_measure(cube, area_cube, 'area') - - cube_with_volume_measure = add_cell_measure(cube, volume_cube, 'volume) - - cube_with_ancillary_sftlf = add_ancillary_variable(cube, sftlf_cube) - - cube_with_ancillary_sftgif = add_ancillary_variable(cube, sftgif_cube) - - Details on the arguments needed for each step can be found in the following sections. - .. _Vertical interpolation: Vertical interpolation @@ -591,11 +597,6 @@ This function requires a land or sea area fraction `ancillary variable`_. This supplementary variable, either ``sftlf`` or ``sftof``, should be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. @@ -642,12 +643,6 @@ but if it is not available it will compute a mask based on This supplementary variable, either ``sftlf`` or ``sftof``, can be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - - If the corresponding ancillary variable is not available (which is the case for some models and almost all observational datasets), the preprocessor attempts to mask the data using Natural Earth mask files (that are @@ -679,11 +674,6 @@ This function requires a land ice area fraction `ancillary variable`_. This supplementary variable ``sftgif`` should be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.mask_landseaice`. Glaciated masking @@ -754,6 +744,8 @@ After ``mask_multimodel``, all involved datasets have an identical mask. See also :func:`esmvalcore.preprocessor.mask_multimodel`. +.. _threshold_masking: + Minimum, maximum and interval masking ------------------------------------- @@ -1031,7 +1023,7 @@ statistics. This grouping is performed taking into account the dataset tags However, they should typically be computed earlier in the workflow. Moreover, because multiple ensemble members of the same model are typically more consistent/homogeneous than datasets from different models, the implementation -is more straigtforward and can benefit from lazy evaluation and more efficient +is more straightforward and can benefit from lazy evaluation and more efficient computation. The preprocessor takes a list of statistics as input: @@ -1043,10 +1035,22 @@ The preprocessor takes a list of statistics as input: ensemble_statistics: statistics: [mean, median] +Additional keyword arguments can be given by using a dictionary: + +.. code-block:: yaml + + preprocessors: + example_preprocessor: + ensemble_statistics: + statistics: + - operator: percentile + percent: 20 + - operator: median + This preprocessor function exposes the iris analysis package, and works with all (capitalized) statistics from the :mod:`iris.analysis` package -that can be executed without additional arguments (e.g. percentiles are not -supported because it requires additional keywords: percentile.). +that can be executed without additional arguments. +See :ref:`stat_preprocs` for more details on supported statistics. Note that ``ensemble_statistics`` will not return the single model and ensemble files, only the requested ensemble statistics results. @@ -1087,11 +1091,12 @@ set-up, initial conditions, forcings and implementation; comparing model data to observational data, these biases have a significantly lower statistical impact when using a multi-model ensemble. ESMValCore has the capability of computing a number of multi-model statistical measures: using the preprocessor module -``multi_model_statistics`` will enable the user to ask for either a multi-model -``mean``, ``median``, ``max``, ``min``, ``std_dev``, and / or ``pXX.YY`` with a set -of argument parameters passed to ``multi_model_statistics``. Percentiles can be -specified like ``p1.5`` or ``p95``. The decimal point will be replaced by a dash -in the output file. +``multi_model_statistics`` will enable the user for example to ask for either a multi-model +``mean``, ``median``, ``max``, ``min``, ``std_dev``, and / or ``percentile`` +with a set of argument parameters passed to ``multi_model_statistics``. +See :ref:`stat_preprocs` for more details on supported statistics. +Percentiles can be specified with additional keyword arguments using the syntax +``statistics: [{operator: percentile, percent: xx}]``. Restrictive computation is also available by excluding any set of models that the user will not want to include in the statistics (by setting ``exclude: @@ -1132,6 +1137,14 @@ and `ptop`) are always removed. exclude: [NCEP-NCAR-R1] keep_input_datasets: false ignore_scalar_coords: true + multi_model_percentiles_5_95: + multi_model_statistics: + span: overlap + statistics: + - operator: percentile + percent: 5 + - operator: percentile + percent: 95 Multi-model statistics also supports a ``groupby`` argument. You can group by any dataset key (``project``, ``experiment``, etc.) or a combination of keys in a list. You can @@ -1183,14 +1196,6 @@ Note that those datasets can be excluded if listed in the ``exclude`` option. See also :func:`esmvalcore.preprocessor.multi_model_statistics`. -.. note:: - - The multi-model array operations can be rather memory-intensive (since they - are not performed lazily as yet). The Section on :ref:`Memory use` details - the memory intake for different run scenarios, but as a thumb rule, for the - multi-model preprocessor, the expected maximum memory intake could be - approximated as the number of datasets multiplied by the average size in - memory for one dataset. .. _time operations: @@ -1294,11 +1299,14 @@ See also :func:`esmvalcore.preprocessor.extract_month`. This function produces statistics at a x-hourly frequency. Parameters: - * every_n_hours: frequency to use to compute the statistics. Must be a divisor of - 24. - - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max' and 'sum'. Default is 'mean' + * `hour`: Number of hours per period. + Must be a divisor of 24, i.e., (1, 2, 3, 4, 6, 8, 12). + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.hourly_statistics`. @@ -1310,8 +1318,12 @@ See also :func:`esmvalcore.preprocessor.hourly_statistics`. This function produces statistics for each day in the dataset. Parameters: - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.daily_statistics`. @@ -1323,8 +1335,12 @@ See also :func:`esmvalcore.preprocessor.daily_statistics`. This function produces statistics for each month in the dataset. Parameters: - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.monthly_statistics`. @@ -1340,14 +1356,17 @@ looking at the DJF field, but your datasets starts on January 1st, the first DJF field will only contain data from January and February. We recommend using the extract_time to start the dataset from the following -December and remove such biased initial datapoints. +December and remove such biased initial data points. Parameters: - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' - - * seasons: seasons to build statistics. - Default is '[DJF, MAM, JJA, SON]' + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * `seasons`: Seasons to build statistics. + Default is ``'[DJF, MAM, JJA, SON]'``. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.seasonal_statistics`. @@ -1359,8 +1378,12 @@ See also :func:`esmvalcore.preprocessor.seasonal_statistics`. This function produces statistics for each year. Parameters: - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.annual_statistics`. @@ -1372,8 +1395,12 @@ See also :func:`esmvalcore.preprocessor.annual_statistics`. This function produces statistics for each decade. Parameters: - * operator: operation to apply. Accepted values are 'mean', - 'median', 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.decadal_statistics`. @@ -1387,16 +1414,25 @@ This function produces statistics for the whole dataset. It can produce scalars statistics. Parameters: - * operator: operation to apply. Accepted values are 'mean', 'median', - 'std_dev', 'min', 'max', 'sum' and 'rms'. Default is 'mean' - - * period: define the granularity of the statistics: get values for the + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + Default is `mean`. + * `period`: Define the granularity of the statistics: get values for the full period, for each month, day of year or hour of day. - Available periods: 'full', 'season', 'seasonal', 'monthly', 'month', - 'mon', 'daily', 'day', 'hourly', 'hour', 'hr'. Default is 'full' + Available periods: `full`, `season`, `seasonal`, `monthly`, `month`, + `mon`, `daily`, `day`, `hourly`, `hour`, `hr`. Default is `full`. + * `seasons`: if period 'seasonal' or 'season' allows to set custom seasons. + Default is ``'[DJF, MAM, JJA, SON]'``. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. - * seasons: if period 'seasonal' or 'season' allows to set custom seasons. - Default is '[DJF, MAM, JJA, SON]' +.. note:: + Some operations are weighted by the time coordinate by default, i.e., the + length of the time intervals. + See :ref:`stat_preprocs` for more details on supported statistics. + For `sum`, the units of the resulting cube are multiplied by the + corresponding time units (e.g., days). Examples: * Monthly climatology: @@ -1404,8 +1440,8 @@ Examples: .. code-block:: yaml climate_statistics: - operator: mean - period: month + operator: mean + period: month * Daily maximum for the full period: @@ -1423,6 +1459,15 @@ Examples: operator: min period: full + * 80% percentile for each month: + + .. code-block:: yaml + + climate_statistics: + period: month + operator: percentile + percent: 80 + See also :func:`esmvalcore.preprocessor.climate_statistics`. .. _resample_time: @@ -1653,6 +1698,7 @@ Examples: See also :func:`esmvalcore.preprocessor.extract_coordinate_points`. +.. _extract_region: ``extract_region`` ------------------ @@ -1839,12 +1885,12 @@ but also mountains or other geographical locations. Note that this function's geolocator application needs a working internet connection. -Parameters - * ``cube``: the input dataset cube to extract a point from. - * ``location``: the reference location. Examples: 'mount everest', - 'romania', 'new york, usa'. Raises ValueError if none supplied. - * ``scheme`` : interpolation scheme. ``'linear'`` or ``'nearest'``. - There is no default, raises ValueError if none supplied. +Parameters: + * `cube`: the input dataset cube to extract a point from. + * `location`: the reference location. Examples: 'mount everest', + 'romania', 'new york, usa'. Raises ValueError if none supplied. + * `scheme` : interpolation scheme. `linear` or `nearest`. + There is no default, raises ValueError if none supplied. See also :func:`esmvalcore.preprocessor.extract_location`. @@ -1853,9 +1899,14 @@ See also :func:`esmvalcore.preprocessor.extract_location`. -------------------- The function calculates the zonal statistics by applying an operator -along the longitude coordinate. This function takes one argument: +along the longitude coordinate. -* ``operator``: Which operation to apply: mean, std_dev, median, min, max, sum or rms. +Parameters: + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.zonal_means`. @@ -1867,7 +1918,12 @@ The function calculates the meridional statistics by applying an operator along the latitude coordinate. This function takes one argument: -* ``operator``: Which operation to apply: mean, std_dev, median, min, max, sum or rms. +Parameters: + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. See also :func:`esmvalcore.preprocessor.meridional_means`. @@ -1877,28 +1933,27 @@ See also :func:`esmvalcore.preprocessor.meridional_means`. ``area_statistics`` ------------------- -This function calculates the average value over a region - weighted by the cell -areas of the region. This function takes the argument, ``operator``: the name -of the operation to apply. +This function calculates statistics over a region. +It takes one argument, ``operator``, which is the name of the operation to +apply. This function can be used to apply several different operations in the -horizontal plane: mean, standard deviation, median, variance, minimum, maximum and root mean square. - -Note that this function is applied over the entire dataset. If only a specific -region, depth layer or time period is required, then those regions need to be -removed using other preprocessor operations in advance. - -This function requires a cell area `cell measure`_, unless the coordinates of the -input data are regular 1D latitude and longitude coordinates so the cell areas -can be computed. -The required supplementary variable, either ``areacella`` for atmospheric variables -or ``areacello`` for ocean variables, can be attached to the main dataset -as described in :ref:`supplementary_variables`. - -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the user - wishes to input to the function. More details on this are given in :ref:`Fx - variables as cell measures or ancillary variables`. +horizontal plane: for example, mean, sum, standard deviation, median, variance, +minimum, maximum and root mean square. +Some operations are grid cell area weighted by default. +For sums, the units of the resulting cubes are multiplied by m :math:`^2`. +See :ref:`stat_preprocs` for more details on supported statistics. + +Note that this function is applied over the entire dataset. +If only a specific region, depth layer or time period is required, then those +regions need to be removed using other preprocessor operations in advance. + +For weighted statistics, this function requires a cell area `cell measure`_, +unless the coordinates of the input data are regular 1D latitude and longitude +coordinates so the cell areas can be computed internally. +The required supplementary variable, either ``areacella`` for atmospheric +variables or ``areacello`` for ocean variables, can be attached to the main +dataset as described in :ref:`supplementary_variables`. See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -1987,25 +2042,22 @@ See also :func:`esmvalcore.preprocessor.extract_volume`. This function calculates the volume-weighted average across three dimensions, but maintains the time dimension. -This function takes the argument: ``operator``, which defines the operation to +This function takes the argument: `operator`, which defines the operation to apply over the volume. +At the moment, only `mean` is supported. +By default, the `mean` operation is weighted by the grid cell volumes. -This function requires a cell volume `cell measure`_, unless the coordinates of -the input data are regular 1D latitude and longitude coordinates so the cell -volumes can be computed. -The required supplementary variable ``volcello`` can be attached to the main dataset -as described in :ref:`supplementary_variables`. +For weighted statistics, this function requires a cell volume `cell measure`_, +unless the coordinates of the input data are regular 1D latitude and longitude +coordinates so the cell volumes can be computed internally. +The required supplementary variable ``volcello`` can be attached to the main +dataset as described in :ref:`supplementary_variables`. No depth coordinate is required as this is determined by Iris. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in - :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.volume_statistics`. +.. _axis_statistics: ``axis_statistics`` --------------------- @@ -2014,26 +2066,36 @@ This function operates over a given axis, and removes it from the output cube. Takes arguments: - * axis: direction over which the statistics will be performed. - Possible values for the axis are 'x', 'y', 'z', 't'. - * operator: defines the operation to apply over the axis. - Available operator are 'mean', 'median', 'std_dev', 'sum', 'variance', - 'min', 'max', 'rms'. + * `axis`: direction over which the statistics will be performed. + Possible values for the axis are `x`, `y`, `z`, `t`. + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. .. note:: The coordinate associated to the axis over which the operation will be performed must be one-dimensional, as multidimensional coordinates are not supported in this preprocessor. + Some operations are weighted by the corresponding coordinate bounds by + default. + For sums, the units of the resulting cubes are multiplied by the + corresponding coordinate units. + See :ref:`stat_preprocs` for more details on supported statistics. + See also :func:`esmvalcore.preprocessor.axis_statistics`. ``depth_integration`` --------------------- -This function integrates over the depth dimension. This function does a -weighted sum along the `z`-coordinate, and removes the `z` direction of the -output cube. This preprocessor takes no arguments. +This function integrates over the depth dimension. +This function does a weighted sum along the `z`-coordinate, and removes the `z` +direction of the output cube. +This preprocessor takes no arguments. +The units of the resulting cube are multiplied by the `z`-coordinate units. See also :func:`esmvalcore.preprocessor.depth_integration`. @@ -2166,13 +2228,14 @@ One can calculate rolling window statistics using the preprocessor function ``rolling_window_statistics``. This function takes three parameters: -* ``coordinate``: coordinate over which the rolling-window statistics is - calculated. - -* ``operator``: operation to apply. Accepted values are 'mean', 'median', - 'std_dev', 'min', 'max' and 'sum'. - -* ``window_length``: size of the rolling window to use (number of points). + * `coordinate`: Coordinate over which the rolling-window statistics is + calculated. + * `operator`: Operation to apply. + See :ref:`stat_preprocs` for more details on supported statistics. + * `window_length`: size of the rolling window to use (number of points). + * Other parameters are directly passed to the `operator` as keyword + arguments. + See :ref:`stat_preprocs` for more details. This example applied on daily precipitation data calculates two-day rolling precipitation sum. diff --git a/esmvalcore/_citation.py b/esmvalcore/_citation.py index 5637f115b3..510aa8cb42 100644 --- a/esmvalcore/_citation.py +++ b/esmvalcore/_citation.py @@ -101,7 +101,8 @@ def _save_citation_bibtex(product_name, tags, json_urls): entries.add(cmip_citation) citation_entries.extend(sorted(entries)) - with open(f'{product_name}_citation.bibtex', 'w') as file: + with open(f'{product_name}_citation.bibtex', + 'w', encoding='utf-8') as file: file.write('\n'.join(citation_entries)) @@ -126,7 +127,8 @@ def _save_citation_info_txt(product_name, info_urls, other_info): for t in sorted(other_info)) if lines: - with open(f'{product_name}_data_citation_info.txt', 'w') as file: + with open(f'{product_name}_data_citation_info.txt', + 'w', encoding='utf-8') as file: file.write('\n'.join(lines) + '\n') @@ -196,7 +198,7 @@ def _collect_bibtex_citation(tag): """Collect information from bibtex files.""" bibtex_file = DIAGNOSTICS.references / f'{tag}.bibtex' if bibtex_file.is_file(): - entry = bibtex_file.read_text() + entry = bibtex_file.read_text(encoding='utf-8') else: entry = '' logger.warning( diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index da5cdd0262..afb0cc0476 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -64,10 +64,10 @@ def parse_resume(resume, recipe): resume[i] = Path(os.path.expandvars(resume_dir)).expanduser() # Sanity check resume directories: - current_recipe = recipe.read_text() + current_recipe = recipe.read_text(encoding='utf-8') for resume_dir in resume: resume_recipe = resume_dir / 'run' / recipe.name - if current_recipe != resume_recipe.read_text(): + if current_recipe != resume_recipe.read_text(encoding='utf-8'): raise ValueError(f'Only identical recipes can be resumed, but ' f'{resume_recipe} is different from {recipe}') return resume @@ -319,7 +319,7 @@ def __init__(self): self.__setattr__(entry_point.name, entry_point.load()()) def version(self): - """Show versions of all packages that conform ESMValTool. + """Show versions of all packages that form ESMValTool. In particular, this command will show the version ESMValCore and any other package that adds a subcommand to 'esmvaltool' @@ -337,7 +337,6 @@ def run(self, max_datasets=None, max_years=None, skip_nonexistent=None, - offline=None, search_esgf=None, diagnostics=None, check_level=None, @@ -365,15 +364,6 @@ def run(self, Maximum number of years to use. skip_nonexistent: bool, optional If True, the run will not fail if some datasets are not available. - offline: bool, optional - If True, the tool will not download missing data from ESGF. - - .. deprecated:: 2.8.0 - This option has been deprecated in ESMValCore version 2.8.0 and - is scheduled for removal in version 2.10.0. Please use the - options `search_esgf=never` (for `offline=True`) or - `search_esgf=when_missing` (for `offline=False`). These are - exact replacements. search_esgf: str, optional If `never`, disable automatic download of data from the ESGF. If `when_missing`, enable the automatic download of files that are not @@ -405,8 +395,6 @@ def run(self, session['max_datasets'] = max_datasets if max_years is not None: session['max_years'] = max_years - if offline is not None: - session['offline'] = offline if search_esgf is not None: session['search_esgf'] = search_esgf if skip_nonexistent is not None: diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 1772884104..4b1b36cb60 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -3,8 +3,8 @@ import logging import os -import re import subprocess +from inspect import getfullargspec from pprint import pformat from shutil import which from typing import Any, Iterable @@ -12,10 +12,12 @@ import isodate import yamale +import esmvalcore.preprocessor from esmvalcore.exceptions import InputFilesNotFound, RecipeError from esmvalcore.local import _get_start_end_year, _parse_period from esmvalcore.preprocessor import TIME_PREPROCESSORS, PreprocessingTask -from esmvalcore.preprocessor._multimodel import STATISTIC_MAPPING +from esmvalcore.preprocessor._multimodel import _get_operator_and_kwargs +from esmvalcore.preprocessor._shared import get_iris_aggregator from esmvalcore.preprocessor._supplementary_vars import ( PREPROCESSOR_SUPPLEMENTARIES, ) @@ -256,20 +258,6 @@ def extract_shape(settings): "{}".format(', '.join(f"'{k}'".lower() for k in valid[key]))) -def _verify_statistics(statistics, step): - """Raise error if multi-model statistics cannot be verified.""" - valid_names = ['std'] + list(STATISTIC_MAPPING.keys()) - valid_patterns = [r"^(p\d{1,2})(\.\d*)?$"] - - for statistic in statistics: - if not (statistic in valid_names - or re.match(r'|'.join(valid_patterns), statistic)): - raise RecipeError( - "Invalid value encountered for `statistic` in preprocessor " - f"{step}. Valid values are {valid_names} " - f"or patterns matching {valid_patterns}. Got '{statistic}'.") - - def _verify_span_value(span): """Raise error if span argument cannot be verified.""" valid_names = ('overlap', 'full') @@ -305,26 +293,8 @@ def _verify_ignore_scalar_coords(ignore_scalar_coords): f"{ignore_scalar_coords}.") -def _verify_arguments(given, expected): - """Raise error if arguments cannot be verified.""" - for key in given: - if key not in expected: - raise RecipeError( - f"Unexpected keyword argument encountered: {key}. Valid " - f"keywords are: {expected}.") - - def multimodel_statistics_preproc(settings): """Check that the multi-model settings are valid.""" - valid_keys = [ - 'groupby', - 'ignore_scalar_coords', - 'keep_input_datasets', - 'span', - 'statistics', - ] - _verify_arguments(settings.keys(), valid_keys) - span = settings.get('span', None) # optional, default: overlap if span: _verify_span_value(span) @@ -333,10 +303,6 @@ def multimodel_statistics_preproc(settings): if groupby: _verify_groupby(groupby) - statistics = settings.get('statistics', None) # required - if statistics: - _verify_statistics(statistics, 'multi_model_statistics') - keep_input_datasets = settings.get('keep_input_datasets', True) _verify_keep_input_datasets(keep_input_datasets) @@ -346,21 +312,10 @@ def multimodel_statistics_preproc(settings): def ensemble_statistics_preproc(settings): """Check that the ensemble settings are valid.""" - valid_keys = [ - 'ignore_scalar_coords', - 'span', - 'statistics', - ] - _verify_arguments(settings.keys(), valid_keys) - span = settings.get('span', 'overlap') # optional, default: overlap if span: _verify_span_value(span) - statistics = settings.get('statistics', None) - if statistics: - _verify_statistics(statistics, 'ensemble_statistics') - ignore_scalar_coords = settings.get('ignore_scalar_coords', False) _verify_ignore_scalar_coords(ignore_scalar_coords) @@ -456,3 +411,65 @@ def reference_for_bias_preproc(products): f"{len(reference_products):d}{ref_products_str}Please also " f"ensure that the reference dataset is not excluded with the " f"'exclude' option") + + +def statistics_preprocessors(settings: dict) -> None: + """Check options of statistics preprocessors.""" + mm_stats = ( + 'multi_model_statistics', + 'ensemble_statistics', + ) + for (step, step_settings) in settings.items(): + + # For multi-model statistics, we need to check each entry of statistics + if step in mm_stats: + _check_mm_stat(step, step_settings) + + # For other statistics, check optional kwargs for operator + elif '_statistics' in step: + _check_regular_stat(step, step_settings) + + +def _check_regular_stat(step, step_settings): + """Check regular statistics (non-multi-model statistics) step.""" + step_settings = dict(step_settings) + + # Some preprocessors like climate_statistics use default 'mean' for + # operator. If 'operator' is missing for those preprocessors with no + # default, this will be detected in PreprocessorFile.check() later. + operator = step_settings.pop('operator', 'mean') + + # If preprocessor does not exist, do nothing here; this will be detected in + # PreprocessorFile.check() later. + try: + preproc_func = getattr(esmvalcore.preprocessor, step) + except AttributeError: + return + + # Ignore other preprocessor arguments, e.g., 'hours' for hourly_statistics + other_args = getfullargspec(preproc_func).args[1:] + operator_kwargs = { + k: v for (k, v) in step_settings.items() if k not in other_args + } + try: + get_iris_aggregator(operator, **operator_kwargs) + except ValueError as exc: + raise RecipeError( + f"Invalid options for {step}: {exc}" + ) + + +def _check_mm_stat(step, step_settings): + """Check multi-model statistic step.""" + statistics = step_settings.get('statistics', []) + for stat in statistics: + try: + (operator, kwargs) = _get_operator_and_kwargs(stat) + except ValueError as exc: + raise RecipeError(str(exc)) + try: + get_iris_aggregator(operator, **kwargs) + except ValueError as exc: + raise RecipeError( + f"Invalid options for {step}: {exc}" + ) diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index 145c2e95d0..8bd33fd5e9 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -168,11 +168,15 @@ def _group_ensemble_members(dataset_facets: Iterable[Facets]) -> list[Facets]: """ def grouper(facets): - return tuple((k, facets[k]) for k in sorted(facets) if k != 'ensemble') + return sorted( + (f, str(v)) for f, v in facets.items() if f != 'ensemble') result = [] - for group_facets, group in itertools.groupby(dataset_facets, key=grouper): + dataset_facets = sorted(dataset_facets, key=grouper) + for _, group_iter in itertools.groupby(dataset_facets, key=grouper): + group = list(group_iter) ensembles = [f['ensemble'] for f in group if 'ensemble' in f] + group_facets = group[0] if not ensembles: result.append(dict(group_facets)) else: diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index ed858e0c39..1613f6cf6d 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -9,7 +9,6 @@ from copy import deepcopy from itertools import groupby from pathlib import Path -from pprint import pformat from typing import Any, Dict, Iterable, Sequence import yaml @@ -17,16 +16,10 @@ from esmvalcore import __version__, esgf from esmvalcore._provenance import get_recipe_provenance from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet -from esmvalcore.cmor.table import CMOR_TABLES, _update_cmor_facets -from esmvalcore.config import CFG -from esmvalcore.config._config import TASKSEP, get_project_config +from esmvalcore.config._config import TASKSEP from esmvalcore.config._diagnostics import TAGS from esmvalcore.dataset import Dataset -from esmvalcore.exceptions import ( - ESMValCoreDeprecationWarning, - InputFilesNotFound, - RecipeError, -) +from esmvalcore.exceptions import InputFilesNotFound, RecipeError from esmvalcore.local import ( _dates_to_timerange, _get_multiproduct_filename, @@ -43,6 +36,7 @@ PreprocessorFile, ) from esmvalcore.preprocessor._area import _update_shapefile_path +from esmvalcore.preprocessor._multimodel import _get_stat_identifier from esmvalcore.preprocessor._other import _group_products from esmvalcore.preprocessor._regrid import ( _spec_to_latlonvals, @@ -50,10 +44,6 @@ get_reference_levels, parse_cell_spec, ) -from esmvalcore.preprocessor._supplementary_vars import ( - PREPROCESSOR_SUPPLEMENTARIES, -) -from esmvalcore.typing import Facets from . import check from .from_datasets import datasets_to_recipe @@ -213,9 +203,6 @@ def _get_default_settings(dataset): settings = {} - # Configure (deprecated, remove for v2.10.0) load callback - settings['load'] = {'callback': 'default'} - if _derive_needed(dataset): settings['derive'] = { 'short_name': facets['short_name'], @@ -235,177 +222,6 @@ def _get_default_settings(dataset): return settings -def _guess_fx_mip(facets: dict, dataset: Dataset): - """Search mip for fx variable.""" - project = facets.get('project', dataset.facets['project']) - # check if project in config-developer - get_project_config(project) - - tables = CMOR_TABLES[project].tables - - # Get all mips that offer that specific fx variable - mips_with_fx_var = [] - for mip in tables: - if facets['short_name'] in tables[mip]: - mips_with_fx_var.append(mip) - - # List is empty -> no table includes the fx variable - if not mips_with_fx_var: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' not available " - f"in any CMOR table for '{project}'") - - # Iterate through all possible mips and check if files are available; in - # case of ambiguity raise an error - fx_files_for_mips = {} - for mip in mips_with_fx_var: - logger.debug("For fx variable '%s', found table '%s'", - facets['short_name'], mip) - fx_dataset = dataset.copy(**facets) - fx_dataset.supplementaries = [] - fx_dataset.set_facet('mip', mip) - fx_dataset.facets.pop('timerange', None) - fx_files = fx_dataset.files - if fx_files: - logger.debug("Found fx variables '%s':\n%s", facets['short_name'], - pformat(fx_files)) - fx_files_for_mips[mip] = fx_files - - # Dict contains more than one element -> ambiguity - if len(fx_files_for_mips) > 1: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' for dataset " - f"'{dataset.facets['dataset']}' of project '{project}' is " - f"available in more than one CMOR MIP table for " - f"'{project}': {sorted(fx_files_for_mips)}") - - # Dict is empty -> no files found -> handled at later stage - if not fx_files_for_mips: - return mips_with_fx_var[0] - - # Dict contains one element -> ok - mip = list(fx_files_for_mips)[0] - return mip - - -def _set_default_preproc_fx_variables( - dataset: Dataset, - settings: PreprocessorSettings, -) -> None: - """Update `fx_variables` key in preprocessor settings with defaults.""" - default_fx = { - 'area_statistics': { - 'areacella': None, - }, - 'mask_landsea': { - 'sftlf': None, - }, - 'mask_landseaice': { - 'sftgif': None, - }, - 'volume_statistics': { - 'volcello': None, - }, - 'weighting_landsea_fraction': { - 'sftlf': None, - }, - } - if dataset.facets['project'] != 'obs4MIPs': - default_fx['area_statistics']['areacello'] = None - default_fx['mask_landsea']['sftof'] = None - default_fx['weighting_landsea_fraction']['sftof'] = None - - for step, fx_variables in default_fx.items(): - if step in settings and 'fx_variables' not in settings[step]: - settings[step]['fx_variables'] = fx_variables - - -def _get_supplementaries_from_fx_variables( - settings: PreprocessorSettings -) -> list[Facets]: - """Read supplementary facets from `fx_variables` in preprocessor.""" - supplementaries = [] - for step, kwargs in settings.items(): - allowed = PREPROCESSOR_SUPPLEMENTARIES.get(step, - {}).get('variables', []) - if fx_variables := kwargs.get('fx_variables'): - - if isinstance(fx_variables, list): - result: dict[str, Facets] = {} - for fx_variable in fx_variables: - if isinstance(fx_variable, str): - # Legacy legacy method of specifying fx variable - short_name = fx_variable - result[short_name] = {} - elif isinstance(fx_variable, dict): - short_name = fx_variable['short_name'] - result[short_name] = fx_variable - fx_variables = result - - for short_name, facets in fx_variables.items(): - if short_name not in allowed: - raise RecipeError( - f"Preprocessor function '{step}' does not support " - f"supplementary variable '{short_name}'") - if facets is None: - facets = {} - facets['short_name'] = short_name - supplementaries.append(facets) - - return supplementaries - - -def _get_legacy_supplementary_facets( - dataset: Dataset, - settings: PreprocessorSettings, -) -> list[Facets]: - """Load the supplementary dataset facets from the preprocessor settings.""" - # First update `fx_variables` in preprocessor settings with defaults - _set_default_preproc_fx_variables(dataset, settings) - - supplementaries = _get_supplementaries_from_fx_variables(settings) - - # Guess the ensemble and mip if they is not specified - for facets in supplementaries: - if 'ensemble' not in facets and dataset.facets['project'] == 'CMIP5': - facets['ensemble'] = 'r0i0p0' - if 'mip' not in facets: - facets['mip'] = _guess_fx_mip(facets, dataset) - return supplementaries - - -def _add_legacy_supplementary_datasets(dataset: Dataset, settings): - """Update fx settings depending on the needed method.""" - if not dataset.session['use_legacy_supplementaries']: - return - if dataset.supplementaries: - # Supplementaries have been defined in the recipe. - # Just remove any skipped supplementaries (they have been kept so we - # know that supplementaries have been defined in the recipe). - dataset.supplementaries = [ - ds for ds in dataset.supplementaries - if not ds.facets.get('skip', False) - ] - return - - logger.debug("Using legacy method to add supplementaries to %s", dataset) - - legacy_ds = dataset.copy() - for facets in _get_legacy_supplementary_facets(dataset, settings): - legacy_ds.add_supplementary(**facets) - - for supplementary_ds in legacy_ds.supplementaries: - _update_cmor_facets(supplementary_ds.facets, override=True) - if supplementary_ds.files: - dataset.supplementaries.append(supplementary_ds) - - dataset._fix_fx_exp() - - # Remove preprocessor keyword argument `fx_variables` - for kwargs in settings.values(): - kwargs.pop('fx_variables', None) - - def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { @@ -598,9 +414,11 @@ def _update_multiproduct(input_products, order, preproc_dir, step): for identifier, products in _group_products(products, by_key=grouping): common_attributes = _get_common_attributes(products, settings) - for statistic in settings.get('statistics', []): + statistics = settings.get('statistics', []) + for statistic in statistics: statistic_attributes = dict(common_attributes) - statistic_attributes[step] = _get_tag(step, identifier, statistic) + stat_id = _get_stat_identifier(statistic) + statistic_attributes[step] = _get_tag(step, identifier, stat_id) statistic_attributes.setdefault('alias', statistic_attributes[step]) statistic_attributes.setdefault('dataset', @@ -614,7 +432,7 @@ def _update_multiproduct(input_products, order, preproc_dir, step): ) # Note that ancestors is set when running the preprocessor func. output_products.add(statistic_product) relevant_settings['output_products'][identifier][ - statistic] = statistic_product + stat_id] = statistic_product return output_products, relevant_settings @@ -687,7 +505,6 @@ def _get_preprocessor_products( _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(dataset.facets, settings) _update_preproc_functions(settings, dataset, datasets, missing_vars) - _add_legacy_supplementary_datasets(dataset, settings) check.preprocessor_supplementaries(dataset, settings) input_datasets = _get_input_datasets(dataset) missing = _check_input_files(input_datasets) @@ -820,6 +637,7 @@ def _update_preproc_functions(settings, dataset, datasets, missing_vars): _update_regrid_time(dataset, settings) if dataset.facets.get('frequency') == 'fx': check.check_for_temporal_preprocs(settings) + check.statistics_preprocessors(settings) def _get_preprocessor_task(datasets, profiles, task_name): @@ -893,7 +711,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._preprocessors = raw_recipe.get('preprocessors', {}) if 'default' not in self._preprocessors: self._preprocessors['default'] = {} - self._set_use_legacy_supplementaries() self.datasets = Dataset.from_recipe(recipe_file, session) self.diagnostics = self._initialize_diagnostics( raw_recipe['diagnostics']) @@ -905,42 +722,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._log_recipe_errors(exc) raise - def _set_use_legacy_supplementaries(self): - """Automatically determine if legacy supplementaries are used.""" - names = set() - steps = set() - for name, profile in self._preprocessors.items(): - for step, kwargs in profile.items(): - if isinstance(kwargs, dict) and 'fx_variables' in kwargs: - names.add(name) - steps.add(step) - if self.session['use_legacy_supplementaries'] is False: - kwargs.pop('fx_variables') - if names: - warnings.warn( - ESMValCoreDeprecationWarning( - "Encountered 'fx_variables' argument in preprocessor(s) " - f"{sorted(names)}, function(s) {sorted(steps)}. The " - "'fx_variables' argument is deprecated and will stop " - "working in v2.10. Please remove it and if automatic " - "definition of supplementary variables does not work " - "correctly, specify the supplementary variables in the " - "recipe as described in https://docs.esmvaltool.org/" - "projects/esmvalcore/en/latest/recipe/preprocessor.html" - "#ancillary-variables-and-cell-measures")) - if self.session['use_legacy_supplementaries'] is None: - logger.info("Running with --use-legacy-supplementaries=True") - self.session['use_legacy_supplementaries'] = True - - # Also adapt the global config if necessary because it is used to check - # if mismatching shapes should be ignored when attaching - # supplementary variables in `esmvalcore.preprocessor. - # _supplementary_vars.add_supplementary_variables` to avoid having to - # introduce a new function argument that is immediately deprecated. - session_use_legacy_supp = self.session['use_legacy_supplementaries'] - if session_use_legacy_supp is not None: - CFG['use_legacy_supplementaries'] = session_use_legacy_supp - def _log_recipe_errors(self, exc): """Log a message with recipe errors.""" logger.error(exc.message) diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 06423cbed6..56d9d44221 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -288,14 +288,13 @@ def _get_dataset_facets_from_recipe( ), ) - if not session['use_legacy_supplementaries']: - preprocessor = facets.get('preprocessor', 'default') - settings = profiles.get(preprocessor, {}) - _append_missing_supplementaries(supplementaries, facets, settings) - supplementaries = [ - facets for facets in supplementaries - if not facets.pop('skip', False) - ] + preprocessor = facets.get('preprocessor', 'default') + settings = profiles.get(preprocessor, {}) + _append_missing_supplementaries(supplementaries, facets, settings) + supplementaries = [ + facets for facets in supplementaries + if not facets.pop('skip', False) + ] return facets, supplementaries diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 01d7861f28..04200371cd 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -119,7 +119,7 @@ def _log_resource_usage(): """Write resource usage to file.""" process = psutil.Process(pid) start_time = time.time() - with open(filename, 'w') as file: + with open(filename, 'w', encoding='utf-8') as file: for msg, max_mem in _get_resource_usage(process, start_time, children): file.write(msg) @@ -219,7 +219,7 @@ def _ncl_type(value): 'end if\n'.format(var_name=var_name)) lines.append(_py2ncl(value, var_name)) - with open(filename, mode) as file: + with open(filename, mode, encoding='utf-8') as file: file.write('\n'.join(lines)) file.write('\n') @@ -301,7 +301,7 @@ def __init__(self, prev_preproc_dir, preproc_dir, name): # Reconstruct output prev_metadata_file = prev_preproc_dir / 'metadata.yml' - with prev_metadata_file.open('rb') as file: + with prev_metadata_file.open('r', encoding='utf-8') as file: prev_metadata = yaml.safe_load(file) products = set() @@ -323,7 +323,7 @@ def _run(self, _): # Write metadata to file self._metadata_file.parent.mkdir(parents=True) - with self._metadata_file.open('w') as file: + with self._metadata_file.open('w', encoding='utf-8') as file: yaml.safe_dump(metadata, file) return [str(self._metadata_file)] @@ -609,7 +609,7 @@ def _collect_provenance(self): logger.debug("Collecting provenance from %s", provenance_file) start = time.time() - table = yaml.safe_load(provenance_file.read_text()) + table = yaml.safe_load(provenance_file.read_text(encoding='utf-8')) ignore = ( 'auxiliary_data_dir', diff --git a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py index 6ee5833c26..6ed2108ff7 100644 --- a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py +++ b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor.check import _get_time_bounds +from esmvalcore.cmor.fixes import get_time_bounds from ..common import OceanFixGrid from ..fix import Fix @@ -33,7 +33,7 @@ def fix_metadata(self, cubes): for cube in cubes: freq = self.extra_facets["frequency"] time = cube.coord("time", dim_coords=True) - bounds = _get_time_bounds(time, freq) + bounds = get_time_bounds(time, freq) if np.any(bounds != time.bounds): time.bounds = bounds logger.warning( diff --git a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py index f2a0a1b7ac..e4c2cc420e 100644 --- a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py +++ b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor.check import _get_time_bounds +from esmvalcore.cmor.fixes import get_time_bounds from ..common import ClFixHybridHeightCoord, OceanFixGrid from ..fix import Fix @@ -39,7 +39,7 @@ def fix_metadata(self, cubes): for cube in cubes: freq = self.extra_facets["frequency"] time = cube.coord("time", dim_coords=True) - bounds = _get_time_bounds(time, freq) + bounds = get_time_bounds(time, freq) if np.any(bounds != time.bounds): time.bounds = bounds logger.warning( diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/aladin63.py similarity index 92% rename from esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py rename to esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/aladin63.py index af0348767a..01ae2bb942 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/cnrm_aladin63.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/aladin63.py @@ -1,4 +1,4 @@ -"""Fixes for rcm CNRM-ALADIN63 driven by CNRM-CERFACS-CNRM-CM5.""" +"""Fixes for rcm ALADIN63 driven by CNRM-CERFACS-CNRM-CM5.""" import numpy as np from esmvalcore.cmor._fixes.cordex.cordex_fixes import TimeLongName as BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/hadrem3_ga7_05.py similarity index 65% rename from esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py rename to esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/hadrem3_ga7_05.py index 0abfc3dff1..1806f3f0a0 100644 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/hadrem3_ga7_05.py @@ -1,4 +1,4 @@ -"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" +"""Fixes for rcm HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( MOHCHadREM3GA705 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/cclm4_8_17.py similarity index 63% rename from esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py rename to esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/cclm4_8_17.py index 51d645f591..88d2123420 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/cclm4_8_17.py @@ -1,4 +1,4 @@ -"""Fixes for rcm CLMcom-CCLM4-8-17 driven by MIROC-MIROC5.""" +"""Fixes for rcm CCLM4-8-17 driven by ICHEC-EC-EARTH.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( CLMcomCCLM4817 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py deleted file mode 100644 index 2f6b9a4d62..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/gerics_remo2015.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Fixes for rcm GERICS-REMO2015 driven by ICHEC-EC-EARTH.""" -from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( - TimeLongName as BaseFix) - -Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/hadrem3_ga7_05.py similarity index 64% rename from esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py rename to esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/hadrem3_ga7_05.py index 15b643aa36..2f816ba55e 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/hadrem3_ga7_05.py @@ -1,4 +1,4 @@ -"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" +"""Fixes for rcm HadREM3-GA7-05 driven by ICHEC-EC-EARTH.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( MOHCHadREM3GA705 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/racmo22e.py similarity index 62% rename from esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py rename to esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/racmo22e.py index cc2440a2b8..9f4cb7a2bc 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/racmo22e.py @@ -1,4 +1,4 @@ -"""Fixes for rcm KNMI-RACMO22E driven by NCC-NorESM1-M.""" +"""Fixes for rcm RACMO22E driven by ICHEC-EC-EARTH.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/rca4.py similarity index 67% rename from esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py rename to esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/rca4.py index 5c9059f74a..114715a5c9 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/rca4.py @@ -1,4 +1,4 @@ -"""Fixes for rcm SMHI-RCA4 driven by NCC-NorESM1-M.""" +"""Fixes for rcm RCA4 driven by ICHEC-EC-EARTH.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/remo2015.py similarity index 62% rename from esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py rename to esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/remo2015.py index 96fd620afe..c50d9a23e2 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/knmi_racmo22e.py +++ b/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/remo2015.py @@ -1,4 +1,4 @@ -"""Fixes for rcm KNMI-RACMO22E driven by ICHEC-EC-EARTH.""" +"""Fixes for rcm REMO2015 driven by ICHEC-EC-EARTH.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/cclm4_8_17.py similarity index 62% rename from esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py rename to esmvalcore/cmor/_fixes/cordex/miroc_miroc5/cclm4_8_17.py index 883e38cecd..ac0460904d 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/clmcom_cclm4_8_17.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/cclm4_8_17.py @@ -1,4 +1,4 @@ -"""Fixes for rcm CLMcom-CCLM4-8-17 driven by ICHEC-EC-EARTH.""" +"""Fixes for rcm CCLM4-8-17 driven by MIROC-MIROC5.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( CLMcomCCLM4817 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/remo2015.py similarity index 62% rename from esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py rename to esmvalcore/cmor/_fixes/cordex/miroc_miroc5/remo2015.py index 9f75b595f9..fbd13bdfab 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/remo2015.py @@ -1,4 +1,4 @@ -"""Fixes for rcm GERICS-REMO2015 driven by MIROC-MIROC5.""" +"""Fixes for rcm REMO2015 driven by MIROC-MIROC5.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/wrf361h.py similarity index 93% rename from esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py rename to esmvalcore/cmor/_fixes/cordex/miroc_miroc5/wrf361h.py index 3767c653ea..f8a69bca9b 100644 --- a/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/uhoh_wrf361h.py +++ b/esmvalcore/cmor/_fixes/cordex/miroc_miroc5/wrf361h.py @@ -1,4 +1,4 @@ -"""Fixes for rcm UHOH-WRF361H driven by MIROC-MIROC5.""" +"""Fixes for rcm WRF361H driven by MIROC-MIROC5.""" import iris from esmvalcore.cmor.fix import Fix diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hadrem3_ga7_05.py similarity index 63% rename from esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py rename to esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hadrem3_ga7_05.py index 004cbc412f..7964a583e0 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hadrem3_ga7_05.py @@ -1,4 +1,4 @@ -"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by MOHC-HadGEM2-ES.""" +"""Fixes for rcm HadREM3-GA7-05 driven by MOHC-HadGEM2-ES.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( MOHCHadREM3GA705 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hirham5.py similarity index 89% rename from esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py rename to esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hirham5.py index fbf5633d13..5dfb91f274 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/dmi_hirham5.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/hirham5.py @@ -1,4 +1,4 @@ -"""Fixes for rcm DMI-HIRHAM driven by MOHC-HadGEM2.""" +"""Fixes for rcm HIRHAM driven by MOHC-HadGEM2.""" from esmvalcore.cmor.fix import Fix diff --git a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/rca4.py similarity index 67% rename from esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py rename to esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/rca4.py index de1d8d6536..740711fcea 100644 --- a/esmvalcore/cmor/_fixes/cordex/ichec_ec_earth/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/rca4.py @@ -1,4 +1,4 @@ -"""Fixes for rcm SMHI-RCA4 driven by ICHEC-EC-EARTH.""" +"""Fixes for rcm RCA4 driven by MOHC-HadGEM2-ES.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/remo2015.py similarity index 66% rename from esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py rename to esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/remo2015.py index 543df3670a..cea145f2d3 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/smhi_rca4.py +++ b/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/remo2015.py @@ -1,4 +1,4 @@ -"""Fixes for rcm SMHI-RCA4 driven by MOHC-HadGEM2-ES.""" +"""Fixes for rcm REMO2015 driven by MOHC-HadGEM2.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/hadrem3_ga7_05.py similarity index 61% rename from esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py rename to esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/hadrem3_ga7_05.py index fd31e9b8a6..4875edfc93 100644 --- a/esmvalcore/cmor/_fixes/cordex/cnrm_cerfacs_cnrm_cm5/mohc_hadrem3_ga7_05.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/hadrem3_ga7_05.py @@ -1,4 +1,4 @@ -"""Fixes for rcm MOHC-HadREM3-GA7-05 driven by CNRM-CERFACS-CNRM-CM5.""" +"""Fixes for rcm HadREM3-GA7-05 driven by MPI-M-MPI-ESM-LR.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( MOHCHadREM3GA705 as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py deleted file mode 100644 index 9a92874b03..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/ictp_regcm4_6.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Fixes for rcm ICTP-RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" -from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( - TimeLongName as BaseFix) - -Pr = BaseFix - -Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py deleted file mode 100644 index ab944112b3..0000000000 --- a/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/knmi_racmo22e.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Fixes for rcm KNMI-RACMO22E driven by MPI-M-MPI-ESM-LR.""" -from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( - TimeLongName as BaseFix) - -Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/racmo22e.py similarity index 62% rename from esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py rename to esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/racmo22e.py index b1d118cd38..676e0dfc47 100644 --- a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/racmo22e.py @@ -1,4 +1,4 @@ -"""Fixes for rcm GERICS-REMO2015 driven by NCC-NorESM1-M.""" +"""Fixes for rcm RACMO22E driven by MPI-M-MPI-ESM-LR.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/regcm4_6.py similarity index 65% rename from esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py rename to esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/regcm4_6.py index 85bb2e352c..f863ed1712 100644 --- a/esmvalcore/cmor/_fixes/cordex/mohc_hadgem2_es/gerics_remo2015.py +++ b/esmvalcore/cmor/_fixes/cordex/mpi_m_mpi_esm_lr/regcm4_6.py @@ -1,4 +1,4 @@ -"""Fixes for rcm GERICS-REMO2015 driven by MOHC-HadGEM2.""" +"""Fixes for rcm RegCM4-6 driven by MPI-M-MPI-ESM-LR.""" from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( TimeLongName as BaseFix) diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/hadrem3_ga7_05.py similarity index 100% rename from esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/mohc_hadrem3_ga7_05.py rename to esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/hadrem3_ga7_05.py diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/racmo22e.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/racmo22e.py new file mode 100644 index 0000000000..e9e2e38734 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/racmo22e.py @@ -0,0 +1,5 @@ +"""Fixes for rcm RACMO22E driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/rca4.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/rca4.py new file mode 100644 index 0000000000..58d14599da --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/rca4.py @@ -0,0 +1,7 @@ +"""Fixes for rcm RCA4 driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix + +Tas = BaseFix diff --git a/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/remo2015.py b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/remo2015.py new file mode 100644 index 0000000000..9722263d70 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cordex/ncc_noresm1_m/remo2015.py @@ -0,0 +1,5 @@ +"""Fixes for rcm REMO2015 driven by NCC-NorESM1-M.""" +from esmvalcore.cmor._fixes.cordex.cordex_fixes import ( + TimeLongName as BaseFix) + +Pr = BaseFix diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 2deeb0d6d3..a6156a231c 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -3,17 +3,35 @@ import importlib import inspect +import logging import tempfile +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +import numpy as np +from cf_units import Unit +from iris.coords import Coord, CoordExtent from iris.cube import Cube, CubeList - -from ..table import CMOR_TABLES +from iris.exceptions import UnitConversionError +from iris.util import reverse + +from esmvalcore.cmor._utils import ( + _get_alternative_generic_lev_coord, + _get_generic_lev_coord_names, + _get_new_generic_level_coord, + _get_simplified_calendar, + _get_single_cube, + _is_unstructured_grid, +) +from esmvalcore.cmor.fixes import get_time_bounds +from esmvalcore.cmor.table import get_var_info if TYPE_CHECKING: - from ...config import Session - from ..table import VariableInfo + from esmvalcore.cmor.table import CoordinateInfo, VariableInfo + from esmvalcore.config import Session + +logger = logging.getLogger(__name__) class Fix: @@ -24,19 +42,23 @@ def __init__( vardef: VariableInfo, extra_facets: Optional[dict] = None, session: Optional[Session] = None, + frequency: Optional[str] = None, ) -> None: """Initialize fix object. Parameters ---------- vardef: - CMOR table entry. + CMOR table entry of the variable. extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. session: Current session which includes configuration and directory information. + frequency: + Expected frequency of the variable. If not given, use the one from + the CMOR table entry of the variable. """ self.vardef = vardef @@ -44,6 +66,9 @@ def __init__( extra_facets = {} self.extra_facets = extra_facets self.session = session + if frequency is None and self.vardef is not None: + frequency = self.vardef.frequency + self.frequency = frequency def fix_file( self, @@ -76,7 +101,7 @@ def fix_file( """ return filepath - def fix_metadata(self, cubes: CubeList) -> CubeList: + def fix_metadata(self, cubes: Sequence[Cube]) -> Sequence[Cube]: """Apply fixes to the metadata of the cube. Changes applied here must not require data loading. @@ -90,7 +115,7 @@ def fix_metadata(self, cubes: CubeList) -> CubeList: Returns ------- - iris.cube.CubeList + Iterable[iris.cube.Cube] Fixed cubes. They can be different instances. """ @@ -163,6 +188,7 @@ def get_fixes( short_name: str, extra_facets: Optional[dict] = None, session: Optional[Session] = None, + frequency: Optional[str] = None, ) -> list: """Get the fixes that must be applied for a given dataset. @@ -177,6 +203,8 @@ def get_fixes( before checking because it is not possible to use the character '-' in python names. + In addition, generic fixes for all datasets are added. + Parameters ---------- project: @@ -193,6 +221,9 @@ def get_fixes( session: Current session which includes configuration and directory information. + frequency: + Expected frequency of the variable. If not given, use the one from + the CMOR table entry of the variable. Returns ------- @@ -200,8 +231,7 @@ def get_fixes( Fixes to apply for the given data. """ - cmor_table = CMOR_TABLES[project] - vardef = cmor_table.get_variable(mip, short_name) + vardef = get_var_info(project, mip, short_name) project = project.replace('-', '_').lower() dataset = dataset.replace('-', '_').lower() @@ -238,9 +268,24 @@ def get_fixes( ) for fix_name in (short_name, mip.lower(), 'allvars'): if fix_name in classes: - fixes.append(classes[fix_name]( - vardef, extra_facets=extra_facets, session=session - )) + fixes.append( + classes[fix_name]( + vardef, + extra_facets=extra_facets, + session=session, + frequency=frequency, + ) + ) + + # Always perform generic fixes for all datasets + fixes.append( + GenericFix( + vardef, # type: ignore + extra_facets=extra_facets, + session=session, + frequency=frequency, + ) + ) return fixes @@ -277,3 +322,579 @@ def get_fixed_filepath( else: output_dir.mkdir(parents=True, exist_ok=True) return output_dir / Path(filepath).name + + +class GenericFix(Fix): + """Class providing generic fixes for all datasets.""" + + def fix_metadata(self, cubes: Sequence[Cube]) -> CubeList: + """Fix cube metadata. + + Parameters + ---------- + cubes: + Cubes to be fixed. + + Returns + ------- + CubeList + Fixed cubes. + + """ + # Make sure the this fix also works when no extra_facets are given + if 'project' in self.extra_facets and 'dataset' in self.extra_facets: + dataset_str = ( + f"{self.extra_facets['project']}:" + f"{self.extra_facets['dataset']}" + ) + else: + dataset_str = None + + # The following fixes are designed to operate on the actual cube that + # corresponds to the variable. Thus, it needs to be assured (possibly + # by prior dataset-specific fixes) that the cubes here contain only one + # relevant cube. + cube = _get_single_cube( + cubes, self.vardef.short_name, dataset_str=dataset_str + ) + + cube = self._fix_standard_name(cube) + cube = self._fix_long_name(cube) + cube = self._fix_psu_units(cube) + cube = self._fix_units(cube) + + cube = self._fix_regular_coord_names(cube) + cube = self._fix_alternative_generic_level_coords(cube) + cube = self._fix_coords(cube) + cube = self._fix_time_coord(cube) + + return CubeList([cube]) + + def fix_data(self, cube: Cube) -> Cube: + """Fix cube data. + + Parameters + ---------- + cube: + Cube to be fixed. + + Returns + ------- + Cube + Fixed cube. + + """ + return cube + + @staticmethod + def _msg_suffix(cube: Cube) -> str: + """Get prefix for log messages.""" + if 'source_file' in cube.attributes: + return f"\n(for file {cube.attributes['source_file']})" + return f"\n(for variable {cube.var_name})" + + def _debug_msg(self, cube: Cube, msg: str, *args) -> None: + """Print debug message.""" + msg += self._msg_suffix(cube) + logger.debug(msg, *args) + + def _warning_msg(self, cube: Cube, msg: str, *args) -> None: + """Print debug message.""" + msg += self._msg_suffix(cube) + logger.warning(msg, *args) + + @staticmethod + def _set_range_in_0_360(array: np.ndarray) -> np.ndarray: + """Convert longitude coordinate to [0, 360].""" + return (array + 360.0) % 360.0 + + def _reverse_coord(self, cube: Cube, coord: Coord) -> tuple[Cube, Coord]: + """Reverse cube along a given coordinate.""" + if coord.ndim == 1: + cube = reverse(cube, cube.coord_dims(coord)) + coord = cube.coord(var_name=coord.var_name) + if coord.has_bounds(): + bounds = coord.core_bounds() + right_bounds = bounds[:-2, 1] + left_bounds = bounds[1:-1, 0] + if np.all(right_bounds != left_bounds): + coord.bounds = np.fliplr(bounds) + self._debug_msg( + cube, + "Coordinate %s values have been reversed", + coord.var_name, + ) + return (cube, coord) + + def _get_effective_units(self) -> str: + """Get effective units.""" + if self.vardef.units.lower() == 'psu': + return '1' + return self.vardef.units + + def _fix_units(self, cube: Cube) -> Cube: + """Fix cube units.""" + if self.vardef.units: + units = self._get_effective_units() + + # We use str(cube.units) in the following to catch `degrees` != + # `degrees_north` + if str(cube.units) != units: + old_units = cube.units + try: + cube.convert_units(units) + except (ValueError, UnitConversionError): + self._warning_msg( + cube, + "Failed to convert cube units from '%s' to '%s'", + old_units, + units, + ) + else: + self._warning_msg( + cube, + "Converted cube units from '%s' to '%s'", + old_units, + units, + ) + return cube + + def _fix_standard_name(self, cube: Cube) -> Cube: + """Fix standard_name.""" + # Do not change empty standard names + if not self.vardef.standard_name: + return cube + + if cube.standard_name != self.vardef.standard_name: + self._warning_msg( + cube, + "Standard name changed from '%s' to '%s'", + cube.standard_name, + self.vardef.standard_name, + ) + cube.standard_name = self.vardef.standard_name + + return cube + + def _fix_long_name(self, cube: Cube) -> Cube: + """Fix long_name.""" + # Do not change empty long names + if not self.vardef.long_name: + return cube + + if cube.long_name != self.vardef.long_name: + self._warning_msg( + cube, + "Long name changed from '%s' to '%s'", + cube.long_name, + self.vardef.long_name, + ) + cube.long_name = self.vardef.long_name + + return cube + + def _fix_psu_units(self, cube: Cube) -> Cube: + """Fix psu units.""" + if cube.attributes.get('invalid_units', '').lower() == 'psu': + cube.units = '1' + cube.attributes.pop('invalid_units') + self._debug_msg(cube, "Units converted from 'psu' to '1'") + return cube + + def _fix_regular_coord_names(self, cube: Cube) -> Cube: + """Fix regular (non-generic-level) coordinate names.""" + for cmor_coord in self.vardef.coordinates.values(): + if cmor_coord.generic_level: + continue # Ignore generic level coordinate in this function + if cube.coords(var_name=cmor_coord.out_name): + continue # Coordinate found -> fine here + if cube.coords(cmor_coord.standard_name): + cube_coord = cube.coord(cmor_coord.standard_name) + self._fix_cmip6_multidim_lat_lon_coord( + cube, cmor_coord, cube_coord + ) + return cube + + def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: + """Fix alternative generic level coordinates.""" + # Avoid overriding existing variable information + cmor_var_coordinates = self.vardef.coordinates.copy() + for (coord_name, cmor_coord) in cmor_var_coordinates.items(): + if not cmor_coord.generic_level: + continue # Ignore non-generic-level coordinates + if not cmor_coord.generic_lev_coords: + continue # Cannot fix anything without coordinate info + + # Extract names of the actual generic level coordinates present in + # the cube (e.g., `hybrid_height`, `standard_hybrid_sigma`) + (standard_name, out_name, name) = _get_generic_lev_coord_names( + cube, cmor_coord + ) + + # Make sure to update variable information with actual generic + # level coordinate if one has been found; this is necessary for + # subsequent fixes + if standard_name: + new_generic_level_coord = _get_new_generic_level_coord( + self.vardef, cmor_coord, coord_name, name + ) + self.vardef.coordinates[coord_name] = new_generic_level_coord + self._debug_msg( + cube, + "Generic level coordinate %s will be checked against %s " + "coordinate information", + coord_name, + name, + ) + + # If a generic level coordinate has been found, we don't need to + # look for alternatives + if standard_name or out_name: + continue + + # Search for alternative coordinates (i.e., regular level + # coordinates); if none found, do nothing + try: + (alternative_coord, + cube_coord) = _get_alternative_generic_lev_coord( + cube, coord_name, self.vardef.table_type + ) + except ValueError: # no alternatives found + continue + + # Fix alternative coord + (cube, cube_coord) = self._fix_coord( + cube, alternative_coord, cube_coord + ) + + return cube + + def _fix_cmip6_multidim_lat_lon_coord( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix CMIP6 multidimensional latitude and longitude coordinates.""" + is_cmip6_multidim_lat_lon = all([ + 'CMIP6' in self.vardef.table_type, + cube_coord.ndim > 1, + cube_coord.standard_name in ('latitude', 'longitude'), + ]) + if is_cmip6_multidim_lat_lon: + self._debug_msg( + cube, + "Multidimensional %s coordinate is not set in CMOR standard, " + "ESMValTool will change the original value of '%s' to '%s' to " + "match the one-dimensional case", + cube_coord.standard_name, + cube_coord.var_name, + cmor_coord.out_name, + ) + cube_coord.var_name = cmor_coord.out_name + + def _fix_coord_units( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix coordinate units.""" + if not cmor_coord.units: + return + + # We use str(cube_coord.units) in the following to catch `degrees` != + # `degrees_north` + if str(cube_coord.units) != cmor_coord.units: + old_units = cube_coord.units + try: + cube_coord.convert_units(cmor_coord.units) + except (ValueError, UnitConversionError): + self._warning_msg( + cube, + "Failed to convert units of coordinate %s from '%s' to " + "'%s'", + cmor_coord.out_name, + old_units, + cmor_coord.units, + ) + else: + self._warning_msg( + cube, + "Coordinate %s units '%s' converted to '%s'", + cmor_coord.out_name, + old_units, + cmor_coord.units, + ) + + def _fix_requested_coord_values( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix requested coordinate values.""" + if not cmor_coord.requested: + return + + # Cannot fix non-1D points + if cube_coord.core_points().ndim != 1: + return + + # Get requested CMOR values + try: + cmor_points = np.array(cmor_coord.requested, dtype=float) + except ValueError: + return + + # Align coordinate points with CMOR values if possible + if cube_coord.core_points().shape != cmor_points.shape: + return + atol = 1e-7 * np.mean(cmor_points) + align_coords = np.allclose( + cube_coord.core_points(), + cmor_points, + rtol=1e-7, + atol=atol, + ) + if align_coords: + cube_coord.points = cmor_points + self._debug_msg( + cube, + "Aligned %s points with CMOR points", + cmor_coord.out_name, + ) + + def _fix_longitude_0_360( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix longitude coordinate to be in [0, 360].""" + if not cube_coord.standard_name == 'longitude': + return (cube, cube_coord) + + # Only apply fixes when values are outside of valid range [0, 360] + inside_0_360 = all([ + cube_coord.core_points().min() >= 0.0, + cube_coord.core_points().max() <= 360.0, + ]) + if inside_0_360: + return (cube, cube_coord) + + # Cannot fix longitudes outside [-360, 720] + if np.any(cube_coord.core_points() < -360.0): + return (cube, cube_coord) + if np.any(cube_coord.core_points() > 720.0): + return (cube, cube_coord) + + # cube.intersection only works for cells with 0 or 2 bounds + # Note: nbounds==0 means there are no bounds given, nbounds==2 + # implies a regular grid with bounds in the grid direction, + # nbounds>2 implies an irregular grid with bounds given as vertices + # of the cell polygon. + if cube_coord.ndim == 1 and cube_coord.nbounds in (0, 2): + lon_extent = CoordExtent(cube_coord, 0.0, 360., True, False) + cube = cube.intersection(lon_extent) + else: + new_lons = cube_coord.core_points().copy() + new_lons = self._set_range_in_0_360(new_lons) + + if cube_coord.core_bounds() is None: + new_bounds = None + else: + new_bounds = cube_coord.core_bounds().copy() + new_bounds = self._set_range_in_0_360(new_bounds) + + new_coord = cube_coord.copy(new_lons, new_bounds) + dims = cube.coord_dims(cube_coord) + cube.remove_coord(cube_coord) + cube.add_aux_coord(new_coord, dims) + new_coord = cube.coord(var_name=cmor_coord.out_name) + self._debug_msg(cube, "Shifted longitude to [0, 360]") + + return (cube, new_coord) + + def _fix_coord_bounds( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix coordinate bounds.""" + if cmor_coord.must_have_bounds != 'yes' or cube_coord.has_bounds(): + return + + # Skip guessing bounds for unstructured grids + if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + 'latitude', 'longitude'): + self._debug_msg( + cube, + "Will not guess bounds for coordinate %s of unstructured grid", + cube_coord.var_name, + ) + return + + try: + cube_coord.guess_bounds() + self._warning_msg( + cube, + "Added guessed bounds to coordinate %s", + cube_coord.var_name, + ) + except ValueError as exc: + self._warning_msg( + cube, + "Cannot guess bounds for coordinate %s: %s", + cube.var_name, + str(exc), + ) + + def _fix_coord_direction( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix coordinate direction (increasing vs. decreasing).""" + # Skip fix for a variety of reasons + if cube_coord.ndim > 1: + return (cube, cube_coord) + if cube_coord.dtype.kind == 'U': + return (cube, cube_coord) + if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + 'latitude', 'longitude' + ): + return (cube, cube_coord) + if len(cube_coord.core_points()) == 1: + return (cube, cube_coord) + if not cmor_coord.stored_direction: + return (cube, cube_coord) + + # Fix coordinates with wrong direction + if cmor_coord.stored_direction == 'increasing': + if cube_coord.core_points()[0] > cube_coord.core_points()[1]: + (cube, cube_coord) = self._reverse_coord(cube, cube_coord) + elif cmor_coord.stored_direction == 'decreasing': + if cube_coord.core_points()[0] < cube_coord.core_points()[1]: + (cube, cube_coord) = self._reverse_coord(cube, cube_coord) + + return (cube, cube_coord) + + def _fix_time_units(self, cube: Cube, cube_coord: Coord) -> None: + """Fix time units in cube and attributes.""" + # Fix cube units + old_units = cube_coord.units + cube_coord.convert_units( + Unit( + 'days since 1850-1-1 00:00:00', + calendar=cube_coord.units.calendar, + ) + ) + simplified_cal = _get_simplified_calendar(cube_coord.units.calendar) + cube_coord.units = Unit( + cube_coord.units.origin, calendar=simplified_cal + ) + + # Fix units of time-related cube attributes + attrs = cube.attributes + parent_time = 'parent_time_units' + if parent_time in attrs: + if attrs[parent_time] in 'no parent': + pass + else: + try: + parent_units = Unit(attrs[parent_time], simplified_cal) + except ValueError: + pass + else: + attrs[parent_time] = 'days since 1850-1-1 00:00:00' + + branch_parent = 'branch_time_in_parent' + if branch_parent in attrs: + attrs[branch_parent] = parent_units.convert( + attrs[branch_parent], cube_coord.units) + + branch_child = 'branch_time_in_child' + if branch_child in attrs: + attrs[branch_child] = old_units.convert( + attrs[branch_child], cube_coord.units) + + def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: + """Fix time bounds.""" + times = {'time', 'time1', 'time2', 'time3'} + key = times.intersection(self.vardef.coordinates) + cmor = self.vardef.coordinates[' '.join(key)] + if cmor.must_have_bounds == 'yes' and not cube_coord.has_bounds(): + cube_coord.bounds = get_time_bounds(cube_coord, self.frequency) + self._warning_msg( + cube, + "Added guessed bounds to coordinate %s", + cube_coord.var_name, + ) + + def _fix_time_coord(self, cube: Cube) -> Cube: + """Fix time coordinate.""" + # Make sure to get dimensional time coordinate if possible + if cube.coords('time', dim_coords=True): + cube_coord = cube.coord('time', dim_coords=True) + elif cube.coords('time'): + cube_coord = cube.coord('time') + else: + return cube + + # Cannot fix wrong time that are not references + if not cube_coord.units.is_time_reference(): + return cube + + # Fix time units + self._fix_time_units(cube, cube_coord) + + # Remove time_origin from coordinate attributes + cube_coord.attributes.pop('time_origin', None) + + # Fix time bounds + self._fix_time_bounds(cube, cube_coord) + + return cube + + def _fix_coord( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix non-time coordinate.""" + self._fix_coord_units(cube, cmor_coord, cube_coord) + (cube, cube_coord) = self._fix_longitude_0_360( + cube, cmor_coord, cube_coord + ) + self._fix_coord_bounds(cube, cmor_coord, cube_coord) + (cube, cube_coord) = self._fix_coord_direction( + cube, cmor_coord, cube_coord + ) + self._fix_requested_coord_values(cube, cmor_coord, cube_coord) + return (cube, cube_coord) + + def _fix_coords(self, cube: Cube) -> Cube: + """Fix non-time coordinates.""" + for cmor_coord in self.vardef.coordinates.values(): + + # Cannot fix generic level coords with no unique CMOR information + if cmor_coord.generic_level and not cmor_coord.out_name: + continue + + # Try to get coordinate from cube; if it does not exists, skip + if not cube.coords(var_name=cmor_coord.out_name): + continue + cube_coord = cube.coord(var_name=cmor_coord.out_name) + + # Fixes for time coord are done separately + if cube_coord.var_name == 'time': + continue + + # Fixes + (cube, cube_coord) = self._fix_coord(cube, cmor_coord, cube_coord) + + return cube diff --git a/esmvalcore/cmor/_fixes/shared.py b/esmvalcore/cmor/_fixes/shared.py index 0e4dfead08..e9ae61499f 100644 --- a/esmvalcore/cmor/_fixes/shared.py +++ b/esmvalcore/cmor/_fixes/shared.py @@ -1,6 +1,7 @@ """Shared functions for fixes.""" import logging import os +from datetime import datetime from functools import lru_cache import dask.array as da @@ -9,8 +10,11 @@ import pandas as pd from cf_units import Unit from iris import NameConstraint +from iris.coords import Coord from scipy.interpolate import interp1d +from esmvalcore.iris_helpers import date2num + logger = logging.getLogger(__name__) @@ -103,7 +107,9 @@ def add_plev_from_altitude(cube): ) pressure_coord = iris.coords.AuxCoord(pressure_points, bounds=pressure_bounds, + var_name='plev', standard_name='air_pressure', + long_name='pressure', units='Pa') cube.add_aux_coord(pressure_coord, cube.coord_dims(height_coord)) return @@ -141,7 +147,9 @@ def add_altitude_from_plev(cube): ) altitude_coord = iris.coords.AuxCoord(altitude_points, bounds=altitude_bounds, + var_name='alt', standard_name='altitude', + long_name='altitude', units='m') cube.add_aux_coord(altitude_coord, cube.coord_dims(plev_coord)) return @@ -411,3 +419,92 @@ def fix_ocean_depth_coord(cube): depth_coord.units = 'm' depth_coord.long_name = 'ocean depth coordinate' depth_coord.attributes = {'positive': 'down'} + + +def get_next_month(month: int, year: int) -> tuple[int, int]: + """Get next month and year. + + Parameters + ---------- + month: + Current month. + year: + Current year. + + Returns + ------- + tuple[int, int] + Next month and next year. + + """ + if month != 12: + return month + 1, year + return 1, year + 1 + + +def get_time_bounds(time: Coord, freq: str) -> np.ndarray: + """Get bounds for time coordinate. + + For monthly data, use the first day of the current month and the first day + of the next month. For yearly or decadal data, use 1 January of the current + year and 1 January of the next year or 10 years from the current year. For + other frequencies (daily, 6-hourly, 3-hourly, hourly), half of the + frequency is subtracted/added from the current point in time to get the + bounds. + + Parameters + ---------- + time: + Time coordinate. + freq: + Frequency. + + Returns + ------- + np.ndarray + Time bounds + + Raises + ------ + NotImplementedError + Non-supported frequency is given. + + """ + bounds = [] + dates = time.units.num2date(time.points) + for step, date in enumerate(dates): + month = date.month + year = date.year + if freq in ['mon', 'mo']: + next_month, next_year = get_next_month(month, year) + min_bound = date2num(datetime(year, month, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), + time.units, time.dtype) + elif freq == 'yr': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), + time.units, time.dtype) + elif freq == 'dec': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), + time.units, time.dtype) + else: + delta = { + 'day': 12.0 / 24, + '6hr': 3.0 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, + } + if freq not in delta: + raise NotImplementedError( + f"Cannot guess time bounds for frequency '{freq}'" + ) + point = time.points[step] + min_bound = point - delta[freq] + max_bound = point + delta[freq] + bounds.append([min_bound, max_bound]) + + return np.array(bounds) diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py new file mode 100644 index 0000000000..4b083f19de --- /dev/null +++ b/esmvalcore/cmor/_utils.py @@ -0,0 +1,219 @@ +"""Utilities for CMOR module.""" +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Optional + +from iris.coords import Coord +from iris.cube import Cube +from iris.exceptions import CoordinateNotFoundError + +from esmvalcore.cmor.table import CMOR_TABLES, CoordinateInfo, VariableInfo + +logger = logging.getLogger(__name__) + +_ALTERNATIVE_GENERIC_LEV_COORDS = { + 'alevel': { + 'CMIP5': ['alt40', 'plevs'], + 'CMIP6': ['alt16', 'plev3'], + 'obs4MIPs': ['alt16', 'plev3'], + }, + 'zlevel': { + 'CMIP3': ['pressure'], + }, +} + + +def _get_alternative_generic_lev_coord( + cube: Cube, + coord_name: str, + cmor_table_type: str, +) -> tuple[CoordinateInfo, Coord]: + """Find alternative generic level coordinate in cube. + + Parameters + ---------- + cube: + Cube to be checked. + coord_name: + Name of the generic level coordinate. + cmor_table_type: + CMOR table type, e.g., CMIP3, CMIP5, CMIP6. Note: This is NOT the + project of the dataset, but rather the entry `cmor_type` in + `config-developer.yml`. + + Returns + ------- + tuple[CoordinateInfo, Coord] + Coordinate information from the CMOR tables and the corresponding + coordinate in the cube. + + Raises + ------ + ValueError + No valid alternative generic level coordinate present in cube. + + """ + alternatives_for_coord = _ALTERNATIVE_GENERIC_LEV_COORDS.get( + coord_name, {} + ) + allowed_alternatives = alternatives_for_coord.get(cmor_table_type, []) + + # Check if any of the allowed alternative coordinates is present in the + # cube + for allowed_alternative in allowed_alternatives: + cmor_coord = CMOR_TABLES[cmor_table_type].coords[allowed_alternative] + if cube.coords(var_name=cmor_coord.out_name): + cube_coord = cube.coord(var_name=cmor_coord.out_name) + return (cmor_coord, cube_coord) + + raise ValueError( + f"Found no valid alternative coordinate for generic level coordinate " + f"'{coord_name}'" + ) + + +def _get_generic_lev_coord_names( + cube: Cube, + cmor_coord: CoordinateInfo, +) -> tuple[str | None, str | None, str | None]: + """Try to get names of a generic level coordinate. + + Parameters + ---------- + cube: + Cube to be checked. + cmor_coord: + Coordinate information from the CMOR table with a non-emmpty + `generic_lev_coords` :obj:`dict`. + + Returns + ------- + tuple[str | None, str | None, str | None] + Tuple of `standard_name`, `out_name`, and `name` of the generic level + coordinate present in the cube. Values are ``None`` if generic level + coordinate has not been found in cube. + + """ + standard_name = None + out_name = None + name = None + + # Iterate over all possible generic level coordinates + for coord in cmor_coord.generic_lev_coords.values(): + # First, try to use var_name to find coordinate + if cube.coords(var_name=coord.out_name): + cube_coord = cube.coord(var_name=coord.out_name) + out_name = coord.out_name + if cube_coord.standard_name == coord.standard_name: + standard_name = coord.standard_name + name = coord.name + + # Second, try to use standard_name to find coordinate + elif cube.coords(coord.standard_name): + standard_name = coord.standard_name + name = coord.name + + return (standard_name, out_name, name) + + +def _get_new_generic_level_coord( + var_info: VariableInfo, + generic_level_coord: CoordinateInfo, + generic_level_coord_name: str, + new_coord_name: str | None, +) -> CoordinateInfo: + """Get new generic level coordinate. + + There are a variety of possible options for each generic level coordinate + (e.g., `alevel`) which is actually present in a cube, for example, + `hybrid_height` or `standard_hybrid_sigma`. This function returns the new + coordinate (e.g., `new_coord_name=hybrid_height`) with the relevant + metadata. + + Note + ---- + This alters the corresponding entry of the original generic level + coordinate's `generic_level_coords` attribute (i.e., + ``generic_level_coord.generic_level_coords[new_coord_name]`) in-place! + + Parameters + ---------- + var_info: + CMOR variable information. + generic_level_coord: + Original generic level coordinate. + generic_level_coord_name: + Original name of the generic level coordinate (e.g., `alevel`). + new_coord_name: + Name of the new generic level coordinate (e.g., `hybrid_height`). + + Returns + ------- + CoordinateInfo + New generic level coordinate. + + """ + new_coord = generic_level_coord.generic_lev_coords[new_coord_name] + new_coord.generic_level = True + new_coord.generic_lev_coords = ( + var_info.coordinates[generic_level_coord_name].generic_lev_coords + ) + return new_coord + + +def _get_simplified_calendar(calendar: str) -> str: + """Simplify calendar.""" + calendar_aliases = { + 'all_leap': '366_day', + 'noleap': '365_day', + 'gregorian': 'standard', + } + return calendar_aliases.get(calendar, calendar) + + +def _is_unstructured_grid(cube: Cube) -> bool: + """Check if cube uses unstructured grid.""" + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + pass + else: + if lat.ndim == 1 and (cube.coord_dims(lat) == cube.coord_dims(lon)): + return True + return False + + +def _get_single_cube( + cube_list: Sequence[Cube], + short_name: str, + dataset_str: Optional[str] = None, +) -> Cube: + if len(cube_list) == 1: + return cube_list[0] + cube = None + for raw_cube in cube_list: + if raw_cube.var_name == short_name: + cube = raw_cube + break + + if dataset_str is None: + dataset_str = '' + else: + dataset_str = f' in {dataset_str}' + + if not cube: + raise ValueError( + f"More than one cube found for variable {short_name}{dataset_str} " + f"but none of their var_names match the expected.\nFull list of " + f"cubes encountered: {cube_list}" + ) + logger.warning( + "Found variable %s%s, but there were other present in the file. Those " + "extra variables are usually metadata (cell area, latitude " + "descriptions) that was not saved according to CF-conventions. It is " + "possible that errors appear further on because of this.\nFull list " + "of cubes encountered: %s", short_name, dataset_str, cube_list) + return cube diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 29f0f74e7e..43214168b8 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1,7 +1,12 @@ """Module for checking iris cubes against their CMOR definitions.""" +from __future__ import annotations + import logging -from datetime import datetime +import warnings +from collections.abc import Callable from enum import IntEnum +from functools import cached_property +from typing import Optional import cf_units import iris.coord_categorisation @@ -9,10 +14,18 @@ import iris.exceptions import iris.util import numpy as np +from iris.cube import Cube -from esmvalcore.iris_helpers import date2num - -from .table import CMOR_TABLES +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor._utils import ( + _get_alternative_generic_lev_coord, + _get_generic_lev_coord_names, + _get_new_generic_level_coord, + _get_simplified_calendar, + _is_unstructured_grid, +) +from esmvalcore.cmor.table import get_var_info +from esmvalcore.exceptions import ESMValCoreDeprecationWarning class CheckLevels(IntEnum): @@ -34,49 +47,6 @@ class CheckLevels(IntEnum): """Do not fail for any discrepancy with CMOR standards.""" -def _get_next_month(month, year): - if month != 12: - return month + 1, year - return 1, year + 1 - - -def _get_time_bounds(time, freq): - bounds = [] - dates = time.units.num2date(time.points) - for step, date in enumerate(dates): - month = date.month - year = date.year - if freq in ['mon', 'mo']: - next_month, next_year = _get_next_month(month, year) - min_bound = date2num(datetime(year, month, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), - time.units, time.dtype) - elif freq == 'yr': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), - time.units, time.dtype) - elif freq == 'dec': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), - time.units, time.dtype) - else: - delta = { - 'day': 12 / 24, - '6hr': 3 / 24, - '3hr': 1.5 / 24, - '1hr': 0.5 / 24, - } - point = time.points[step] - min_bound = point - delta[freq] - max_bound = point + delta[freq] - bounds.append([min_bound, max_bound]) - - return np.array(bounds) - - class CMORCheckError(Exception): """Exception raised when a cube does not pass the CMORCheck.""" @@ -84,9 +54,6 @@ class CMORCheckError(Exception): class CMORCheck(): """Class used to check the CMOR-compliance of the data. - It can also fix some minor errors and does some minor data - homogeneization: - Parameters ---------- cube: iris.cube.Cube: @@ -94,13 +61,23 @@ class CMORCheck(): var_info: variables_info.VariableInfo Variable info to check. frequency: str - Expected frequency for the data. + Expected frequency for the data. If not given, use the one from the + variable information. fail_on_error: bool If true, CMORCheck stops on the first error. If false, it collects all possible errors before stopping. automatic_fixes: bool If True, CMORCheck will try to apply automatic fixes for any detected error, if possible. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.fix_metadata`, + :func:`~esmvalcore.preprocessor.fix_data`, or + :meth:`esmvalcore.dataset.Dataset.load` (which automatically + includes the first two functions) instead. Fixes and CMOR checks + have been clearly separated in ESMValCore version 2.10.0. check_level: CheckLevels Level of strictness of the checks. @@ -131,7 +108,6 @@ def __init__(self, self._errors = list() self._warnings = list() self._debug_messages = list() - self._unstructured = None self._cmor_var = var_info if not frequency: @@ -139,47 +115,61 @@ def __init__(self, self.frequency = frequency self.automatic_fixes = automatic_fixes - def _is_unstructured_grid(self): - if self._unstructured is None: - self._unstructured = False - try: - lat = self._cube.coord('latitude') - lon = self._cube.coord('longitude') - except iris.exceptions.CoordinateNotFoundError: - pass - else: - if lat.ndim == 1 and (self._cube.coord_dims(lat) - == self._cube.coord_dims(lon)): - self._unstructured = True - return self._unstructured - - def check_metadata(self, logger=None): + # Deprecate automatic_fixes (remove in v2.12) + if automatic_fixes: + msg = ( + "The option `automatic_fixes` has been deprecated in " + "ESMValCore version 2.10.0 and is scheduled for removal in " + "version 2.12.0. Please use the functions " + "esmvalcore.preprocessor.fix_metadata(), " + "esmvalcore.preprocessor.fix_data(), or " + "esmvalcore.dataset.Dataset.load() (which automatically " + "includes the first two functions) instead. Fixes and CMOR " + "checks have been clearly separated in ESMValCore version " + "2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + + # TODO: remove in v2.12 + + self._generic_fix = GenericFix(var_info, frequency=frequency) + + @cached_property + def _unstructured_grid(self) -> bool: + """Cube uses unstructured grid.""" + return _is_unstructured_grid(self._cube) + + def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube metadata. - Perform all the tests that do not require to have the data in memory. - - It will also report some warnings in case of minor errors and - homogenize some data: - - - Equivalent calendars will all default to the same name. - - Time units will be set to days since 1850-01-01 + It will also report some warnings in case of minor errors. Parameters ---------- - logger: logging.Logger + logger: Given logger. + Returns + ------- + iris.cube.Cube + Checked cube. + Raises ------ CMORCheckError If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. + """ if logger is not None: self._logger = logger + # TODO: remove in v2.12 + if self.automatic_fixes: + [self._cube] = self._generic_fix.fix_metadata([self._cube]) + self._check_var_metadata() self._check_fill_value() self._check_multiple_coords_same_stdname() @@ -196,10 +186,9 @@ def check_metadata(self, logger=None): return self._cube - def check_data(self, logger=None): + def check_data(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube data. - Performs all the tests that require to have the data in memory. Assumes that metadata is correct, so you must call check_metadata prior to this. @@ -207,28 +196,35 @@ def check_data(self, logger=None): Parameters ---------- - logger: logging.Logger + logger: Given logger. + Returns + ------- + iris.cube.Cube + Checked cube. + Raises ------ CMORCheckError If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. + """ if logger is not None: self._logger = logger - if self._cmor_var.units: - units = self._get_effective_units() - if str(self._cube.units) != units: - self._cube.convert_units(units) + # TODO: remove in v2.12 + if self.automatic_fixes: + self._cube = self._generic_fix.fix_data(self._cube) self._check_coords_data() + self.report_debug_messages() self.report_warnings() self.report_errors() + return self._cube def report_errors(self): @@ -251,13 +247,7 @@ def report_errors(self): raise CMORCheckError(msg) def report_warnings(self): - """Report detected warnings to the given logger. - - Parameters - ---------- - logger: logging.Logger - Given logger - """ + """Report detected warnings to the given logger.""" if self.has_warnings(): msg = '\n'.join([ f'There were warnings in variable {self._cube.var_name}:', @@ -268,13 +258,7 @@ def report_warnings(self): self._logger.warning(msg) def report_debug_messages(self): - """Report detected debug messages to the given logger. - - Parameters - ---------- - logger: logging.Logger - Given logger. - """ + """Report detected debug messages to the given logger.""" if self.has_debug_messages(): msg = '\n'.join([ f'There were metadata changes in variable ' @@ -298,49 +282,24 @@ def _check_var_metadata(self): # Check standard_name if self._cmor_var.standard_name: if self._cube.standard_name != self._cmor_var.standard_name: - if self.automatic_fixes: - self.report_warning( - 'Standard name for {} changed from {} to {}', - self._cube.var_name, self._cube.standard_name, - self._cmor_var.standard_name) - self._cube.standard_name = self._cmor_var.standard_name - else: - self.report_error(self._attr_msg, self._cube.var_name, - 'standard_name', - self._cmor_var.standard_name, - self._cube.standard_name) + self.report_error(self._attr_msg, self._cube.var_name, + 'standard_name', + self._cmor_var.standard_name, + self._cube.standard_name) # Check long_name if self._cmor_var.long_name: if self._cube.long_name != self._cmor_var.long_name: - if self.automatic_fixes: - self.report_warning( - 'Long name for {} changed from {} to {}', - self._cube.var_name, self._cube.long_name, - self._cmor_var.long_name) - self._cube.long_name = self._cmor_var.long_name - else: - self.report_error(self._attr_msg, self._cube.var_name, - 'long_name', self._cmor_var.long_name, - self._cube.long_name) + self.report_error(self._attr_msg, self._cube.var_name, + 'long_name', self._cmor_var.long_name, + self._cube.long_name) # Check units - if (self.automatic_fixes and self._cube.attributes.get( - 'invalid_units', '').lower() == 'psu'): - self._cube.units = '1.0' - del self._cube.attributes['invalid_units'] - if self._cmor_var.units: units = self._get_effective_units() if self._cube.units != units: - if not self._cube.units.is_convertible(units): - self.report_error(f'Variable {self._cube.var_name} units ' - f'{self._cube.units} can not be ' - f'converted to {self._cmor_var.units}') - else: - self.report_warning( - f'Variable {self._cube.var_name} units ' - f'{self._cube.units} will be ' - f'converted to {self._cmor_var.units}') + self.report_error(self._attr_msg, self._cube.var_name, + 'units', self._cmor_var.units, + self._cube.units) # Check other variable attributes that match entries in cube.attributes attrs = ('positive', ) @@ -357,6 +316,7 @@ def _check_var_metadata(self): def _get_effective_units(self): """Get effective units.""" + # TODO: remove entire function in v2.12 if self._cmor_var.units.lower() == 'psu': units = '1.0' else: @@ -423,20 +383,7 @@ def _check_dim_names(self): except iris.exceptions.CoordinateNotFoundError: try: coord = self._cube.coord(coordinate.standard_name) - if self._cmor_var.table_type in 'CMIP6' and \ - coord.ndim > 1 and \ - coord.standard_name in ['latitude', 'longitude']: - self.report_debug_message( - 'Multidimensional {0} coordinate is not set ' - 'in CMOR standard. ESMValTool will change ' - 'the original value of {1} to {2} to match ' - 'the one-dimensional case.', - coordinate.standard_name, - coord.var_name, - coordinate.out_name, - ) - coord.var_name = coordinate.out_name - elif coord.standard_name in ['region', 'area_type']: + if coord.standard_name in ['region', 'area_type']: self.report_debug_message( 'Coordinate {0} has var name {1} ' 'instead of {2}. ' @@ -467,33 +414,17 @@ def _check_dim_names(self): def _check_generic_level_dim_names(self, key, coordinate): """Check name of generic level coordinate.""" - standard_name = None - out_name = None - name = None if coordinate.generic_lev_coords: - for coord in coordinate.generic_lev_coords.values(): - try: - cube_coord = self._cube.coord(var_name=coord.out_name) - out_name = coord.out_name - if cube_coord.standard_name == coord.standard_name: - standard_name = coord.standard_name - name = coord.name - except iris.exceptions.CoordinateNotFoundError: - try: - cube_coord = self._cube.coord( - var_name=coord.standard_name) - standard_name = coord.standard_name - name = coord.name - except iris.exceptions.CoordinateNotFoundError: - pass + (standard_name, out_name, name) = _get_generic_lev_coord_names( + self._cube, coordinate + ) if standard_name: if not out_name: self.report_error( f'Generic level coordinate {key} has wrong var_name.') - level = coordinate.generic_lev_coords[name] - level.generic_level = True - level.generic_lev_coords = self._cmor_var.coordinates[ - key].generic_lev_coords + level = _get_new_generic_level_coord( + self._cmor_var, coordinate, key, name + ) self._cmor_var.coordinates[key] = level self.report_debug_message(f'Generic level coordinate {key} ' 'will be checked against ' @@ -506,17 +437,6 @@ def _check_generic_level_dim_names(self, key, coordinate): else: self._check_alternative_dim_names(key) - ALTERNATIVE_GENERIC_LEV_COORDS = { - 'alevel': { - 'CMIP5': ['alt40', 'plevs'], - 'CMIP6': ['alt16', 'plev3'], - 'obs4MIPs': ['alt16', 'plev3'], - }, - 'zlevel': { - 'CMIP3': ['pressure'], - }, - } - def _check_alternative_dim_names(self, key): """Check for viable alternatives to generic level coordinates. @@ -549,42 +469,34 @@ def _check_alternative_dim_names(self, key): For ``cmor_strict=False`` project (like OBS) the check for requested values might be disabled. """ - table_type = self._cmor_var.table_type - alternative_coord = None - allowed_alternatives = self.ALTERNATIVE_GENERIC_LEV_COORDS.get( - key, {}).get(table_type, []) - - # Check if any of the allowed alternative coordinates is present in the - # cube - for allowed_alternative in allowed_alternatives: - coord_info = CMOR_TABLES[table_type].coords[allowed_alternative] - try: - cube_coord = self._cube.coord(var_name=coord_info.out_name) - except iris.exceptions.CoordinateNotFoundError: - pass - else: - if cube_coord.standard_name == coord_info.standard_name: - alternative_coord = coord_info - break - self.report_error( - f"Found alternative coordinate '{coord_info.out_name}' " - f"for generic level coordinate '{key}' with wrong " - f"standard_name '{cube_coord.standard_name}' (expected " - f"'{coord_info.standard_name}')") - break + try: + (alternative_coord, + cube_coord) = _get_alternative_generic_lev_coord( + self._cube, key, self._cmor_var.table_type + ) # No valid alternative coordinate found -> critical error - if alternative_coord is None: + except ValueError: self.report_critical(self._does_msg, key, 'exist') return + # Wrong standard_name -> error + if cube_coord.standard_name != alternative_coord.standard_name: + self.report_error( + f"Found alternative coordinate '{alternative_coord.out_name}' " + f"for generic level coordinate '{key}' with wrong " + f"standard_name {cube_coord.standard_name}' (expected " + f"'{alternative_coord.standard_name}')" + ) + return + # Valid alternative coordinate found -> perform checks on it self.report_warning( f"Found alternative coordinate '{alternative_coord.out_name}' " f"for generic level coordinate '{key}'. Subsequent warnings about " f"levels that are not contained in '{alternative_coord.out_name}' " f"can be safely ignored.") - self._check_coord(alternative_coord, cube_coord, self._cube.var_name) + self._check_coord(alternative_coord, cube_coord, cube_coord.var_name) def _check_coords(self): """Check coordinates.""" @@ -616,6 +528,12 @@ def _check_coords_data(self): except iris.exceptions.CoordinateNotFoundError: continue + # TODO: remove in v2.12 + if self.automatic_fixes: + (self._cube, coord) = self._generic_fix._fix_coord_direction( + self._cube, coordinate, coord + ) + self._check_coord_monotonicity_and_direction( coordinate, coord, var_name) @@ -625,57 +543,24 @@ def _check_coord(self, cmor, coord, var_name): return if cmor.units: if str(coord.units) != cmor.units: - fixed = False - if self.automatic_fixes: - try: - old_unit = coord.units - new_unit = cf_units.Unit(cmor.units, - coord.units.calendar) - coord.convert_units(new_unit) - fixed = True - self.report_warning( - f'Coordinate {coord.var_name} units ' - f'{str(old_unit)} ' - f'converted to {cmor.units}') - except ValueError: - pass - if not fixed: - self.report_critical(self._attr_msg, var_name, 'units', - cmor.units, coord.units) + self.report_critical(self._attr_msg, var_name, 'units', + cmor.units, coord.units) self._check_coord_points(cmor, coord, var_name) def _check_coord_bounds(self, cmor, coord, var_name): if cmor.must_have_bounds == 'yes' and not coord.has_bounds(): - if self.automatic_fixes: - try: - coord.guess_bounds() - except ValueError as ex: - self.report_warning( - 'Can not guess bounds for coordinate {0} ' - 'from var {1}: {2}', coord.var_name, var_name, ex) - else: - self.report_warning( - 'Added guessed bounds to coordinate {0} from var {1}', - coord.var_name, var_name) - else: - self.report_warning( - 'Coordinate {0} from var {1} does not have bounds', - coord.var_name, var_name) + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + coord.var_name, var_name) - def _check_time_bounds(self, freq, time): + def _check_time_bounds(self, time): times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self._cmor_var.coordinates) cmor = self._cmor_var.coordinates[" ".join(key)] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): - if self.automatic_fixes: - time.bounds = _get_time_bounds(time, freq) - self.report_warning( - 'Added guessed bounds to coordinate {0} from var {1}', - time.var_name, self._cmor_var.short_name) - else: - self.report_warning( - 'Coordinate {0} from var {1} does not have bounds', - time.var_name, self._cmor_var.short_name) + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + time.var_name, self._cmor_var.short_name) def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" @@ -684,8 +569,8 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): if coord.dtype.kind == 'U': return - if self._is_unstructured_grid() and \ - coord.standard_name in ['latitude', 'longitude']: + if (self._unstructured_grid and + coord.standard_name in ['latitude', 'longitude']): self.report_debug_message( f'Coordinate {coord.standard_name} appears to belong to ' 'an unstructured grid. Skipping monotonicity and ' @@ -694,118 +579,42 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): if not coord.is_monotonic(): self.report_critical(self._is_msg, var_name, 'monotonic') + if len(coord.core_points()) == 1: return + if cmor.stored_direction: if cmor.stored_direction == 'increasing': if coord.core_points()[0] > coord.core_points()[1]: - if not self.automatic_fixes or coord.ndim > 1: - self.report_critical(self._is_msg, var_name, - 'increasing') - else: - self._reverse_coord(coord) + self.report_critical(self._is_msg, var_name, 'increasing') elif cmor.stored_direction == 'decreasing': if coord.core_points()[0] < coord.core_points()[1]: - if not self.automatic_fixes or coord.ndim > 1: - self.report_critical(self._is_msg, var_name, - 'decreasing') - else: - self._reverse_coord(coord) - - def _reverse_coord(self, coord): - """Reverse coordinate.""" - if coord.ndim == 1: - self._cube = iris.util.reverse(self._cube, - self._cube.coord_dims(coord)) - reversed_coord = self._cube.coord(var_name=coord.var_name) - if reversed_coord.has_bounds(): - bounds = reversed_coord.bounds - right_bounds = bounds[:-2, 1] - left_bounds = bounds[1:-1, 0] - if np.all(right_bounds != left_bounds): - reversed_coord.bounds = np.fliplr(bounds) - coord = reversed_coord - self.report_debug_message(f'Coordinate {coord.var_name} values' - 'have been reversed.') + self.report_critical(self._is_msg, var_name, 'decreasing') def _check_coord_points(self, coord_info, coord, var_name): """Check coordinate points: values, bounds and monotonicity.""" # Check requested coordinate values exist in coord.points self._check_requested_values(coord, coord_info, var_name) - l_fix_coord_value = False - # Check coordinate value ranges if coord_info.valid_min: valid_min = float(coord_info.valid_min) if np.any(coord.core_points() < valid_min): - if coord_info.standard_name == 'longitude' and \ - self.automatic_fixes: - l_fix_coord_value = self._check_longitude_min( - coord, var_name) - else: - self.report_critical(self._vals_msg, var_name, - '< {} ='.format('valid_min'), - valid_min) + self.report_critical(self._vals_msg, var_name, + '< {} ='.format('valid_min'), + valid_min) if coord_info.valid_max: valid_max = float(coord_info.valid_max) if np.any(coord.core_points() > valid_max): - if coord_info.standard_name == 'longitude' and \ - self.automatic_fixes: - l_fix_coord_value = self._check_longitude_max( - coord, var_name) - else: - self.report_critical(self._vals_msg, var_name, - '> {} ='.format('valid_max'), - valid_max) - - if l_fix_coord_value: - # cube.intersection only works for cells with 0 or 2 bounds - # Note: nbounds==0 means there are no bounds given, nbounds==2 - # implies a regular grid with bounds in the grid direction, - # nbounds>2 implies an irregular grid with bounds given as vertices - # of the cell polygon. - if coord.ndim == 1 and coord.nbounds in (0, 2): - lon_extent = iris.coords.CoordExtent(coord, 0.0, 360., True, - False) - self._cube = self._cube.intersection(lon_extent) - else: - new_lons = coord.core_points().copy() - new_lons = self._set_range_in_0_360(new_lons) - if coord.bounds is not None: - new_bounds = coord.bounds.copy() - new_bounds = self._set_range_in_0_360(new_bounds) - else: - new_bounds = None - new_coord = coord.copy(new_lons, new_bounds) - dims = self._cube.coord_dims(coord) - self._cube.remove_coord(coord) - self._cube.add_aux_coord(new_coord, dims) - coord = self._cube.coord(var_name=var_name) + self.report_critical(self._vals_msg, var_name, + '> {} ='.format('valid_max'), + valid_max) + self._check_coord_bounds(coord_info, coord, var_name) self._check_coord_monotonicity_and_direction(coord_info, coord, var_name) - def _check_longitude_max(self, coord, var_name): - if np.any(coord.core_points() > 720): - self.report_critical( - f'{var_name} longitude coordinate has values > 720 degrees') - return False - return True - - def _check_longitude_min(self, coord, var_name): - if np.any(coord.core_points() < -360): - self.report_critical( - f'{var_name} longitude coordinate has values < -360 degrees') - return False - return True - - @staticmethod - def _set_range_in_0_360(array): - """Convert longitude coordinate to [0, 360].""" - return (array + 360.0) % 360.0 - def _check_requested_values(self, coord, coord_info, var_name): """Check requested values.""" if coord_info.requested: @@ -818,17 +627,6 @@ def _check_requested_values(self, coord, coord_info, var_name): cmor_points = np.array(coord_info.requested, dtype=float) except ValueError: cmor_points = coord_info.requested - else: - atol = 1e-7 * np.mean(cmor_points) - if (self.automatic_fixes - and coord.core_points().shape == cmor_points.shape - and np.allclose( - coord.core_points(), - cmor_points, - rtol=1e-7, - atol=atol, - )): - coord.points = cmor_points for point in cmor_points: if point not in coord.core_points(): self.report_warning(self._contain_msg, var_name, @@ -853,12 +651,7 @@ def _check_time_coord(self): self.report_critical(self._does_msg, var_name, 'have time reference units') else: - old_units = coord.units - coord.convert_units( - cf_units.Unit('days since 1850-1-1 00:00:00', - calendar=coord.units.calendar)) - simplified_cal = self._simplify_calendar(coord.units.calendar) - coord.units = cf_units.Unit(coord.units.origin, simplified_cal) + simplified_cal = _get_simplified_calendar(coord.units.calendar) attrs = self._cube.attributes parent_time = 'parent_time_units' if parent_time in attrs: @@ -866,8 +659,7 @@ def _check_time_coord(self): pass else: try: - parent_units = cf_units.Unit(attrs[parent_time], - simplified_cal) + cf_units.Unit(attrs[parent_time], simplified_cal) except ValueError: self.report_warning('Attribute parent_time_units has ' 'a wrong format and cannot be ' @@ -875,17 +667,8 @@ def _check_time_coord(self): 'be added to convert properly ' 'attributes branch_time_in_parent ' 'and branch_time_in_child.') - else: - attrs[parent_time] = 'days since 1850-1-1 00:00:00' - branch_parent = 'branch_time_in_parent' - if branch_parent in attrs: - attrs[branch_parent] = parent_units.convert( - attrs[branch_parent], coord.units) - branch_child = 'branch_time_in_child' - if branch_child in attrs: - attrs[branch_child] = old_units.convert( - attrs[branch_child], coord.units) + # Check frequency tol = 0.001 intervals = {'dec': (3600, 3660), 'day': (1, 1)} freq = self.frequency @@ -941,18 +724,8 @@ def _check_time_coord(self): msg = '{}: Frequency {} does not match input data' self.report_error(msg, var_name, freq) break - self._check_time_bounds(freq, coord) - # remove time_origin from attributes - coord.attributes.pop('time_origin', None) - - @staticmethod - def _simplify_calendar(calendar): - calendar_aliases = { - 'all_leap': '366_day', - 'noleap': '365_day', - 'gregorian': 'standard', - } - return calendar_aliases.get(calendar, calendar) + + self._check_time_bounds(coord) def has_errors(self): """Check if there are reported errors. @@ -1071,30 +844,19 @@ def report_debug_message(self, message, *args): self.report(CheckLevels.DEBUG, message, *args) -def _get_cmor_checker(table, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=CheckLevels.DEFAULT, - automatic_fixes=False): - """Get a CMOR checker/fixer.""" - if table not in CMOR_TABLES: - raise NotImplementedError( - "No CMOR checker implemented for table {}." - "\nThe following options are available: {}".format( - table, ', '.join(CMOR_TABLES))) - - cmor_table = CMOR_TABLES[table] - if table == 'CORDEX' and mip.endswith('hr'): - # CORDEX X-hourly tables define the mip - # as ending in 'h' instead of 'hr'. - mip = mip.replace('hr', 'h') - var_info = cmor_table.get_variable(mip, short_name) - if var_info is None: - var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) - - def _checker(cube): +def _get_cmor_checker( + project: str, + mip: str, + short_name: str, + frequency: None | str = None, + fail_on_error: bool = False, + check_level: CheckLevels = CheckLevels.DEFAULT, + automatic_fixes: bool = False, # TODO: remove in v2.12 +) -> Callable[[Cube], CMORCheck]: + """Get a CMOR checker.""" + var_info = get_var_info(project, mip, short_name) + + def _checker(cube: Cube) -> CMORCheck: return CMORCheck(cube, var_info, frequency=frequency, @@ -1105,105 +867,143 @@ def _checker(cube): return _checker -def cmor_check_metadata(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=CheckLevels.DEFAULT): +def cmor_check_metadata( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if metadata conforms to variable's CMOR definition. None of the checks at this step will force the cube to load the data. Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name. - frequency: str - Data frequency. - check_level: CheckLevels + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - checker = _get_cmor_checker(cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - checker(cube).check_metadata() + checker = _get_cmor_checker( + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = checker(cube).check_metadata() return cube -def cmor_check_data(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=CheckLevels.DEFAULT): +def cmor_check_data( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if data conforms to variable's CMOR definition. - The checks performed at this step require the data in memory. - Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name - frequency: str - Data frequency - check_level: CheckLevels + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - checker = _get_cmor_checker(cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - checker(cube).check_data() + checker = _get_cmor_checker( + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = checker(cube).check_data() return cube -def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): +def cmor_check( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if cube conforms to variable's CMOR definition. - Equivalent to calling cmor_check_metadata and cmor_check_data - consecutively. + Equivalent to calling :func:`cmor_check_metadata` and + :func:`cmor_check_data` consecutively. Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name. - frequency: str - Data frequency. - check_level: enum.IntEnum + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - cmor_check_metadata(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - cmor_check_data(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=check_level) + cube = cmor_check_metadata( + cube, + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = cmor_check_data( + cube, + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) return cube diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 1034435f92..d05af5ef64 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -7,14 +7,17 @@ from __future__ import annotations import logging +import warnings from collections import defaultdict +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Optional -from iris.cube import CubeList +from iris.cube import Cube, CubeList -from ._fixes.fix import Fix -from .check import CheckLevels, _get_cmor_checker +from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor.check import CheckLevels, _get_cmor_checker +from esmvalcore.exceptions import ESMValCoreDeprecationWarning if TYPE_CHECKING: from ..config import Session @@ -31,11 +34,12 @@ def fix_file( output_dir: Path, add_unique_suffix: bool = False, session: Optional[Session] = None, + frequency: Optional[str] = None, **extra_facets, ) -> str | Path: """Fix files before ESMValTool can load them. - This fixes are only for issues that prevent iris from loading the cube or + These fixes are only for issues that prevent iris from loading the cube or that cannot be fixed after the cube is loaded. Original files are not overwritten. @@ -58,6 +62,8 @@ def fix_file( Adds a unique suffix to `output_dir` for thread safety. session: Current session which includes configuration and directory information. + frequency: + Variable's data frequency, if available. **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. @@ -75,6 +81,7 @@ def fix_file( 'project': project, 'dataset': dataset, 'mip': mip, + 'frequency': frequency, }) for fix in Fix.get_fixes(project=project, @@ -82,61 +89,81 @@ def fix_file( mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session): + session=session, + frequency=frequency): file = fix.fix_file( file, output_dir, add_unique_suffix=add_unique_suffix ) return file -def fix_metadata(cubes, - short_name, - project, - dataset, - mip, - frequency=None, - check_level=CheckLevels.DEFAULT, - session: Optional[Session] = None, - **extra_facets): - """Fix cube metadata if fixes are required and check it anyway. +def fix_metadata( + cubes: Sequence[Cube], + short_name: str, + project: str, + dataset: str, + mip: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, + session: Optional[Session] = None, + **extra_facets, +) -> CubeList: + """Fix cube metadata if fixes are required. - This method collects all the relevant fixes for a given variable, applies - them and checks the resulting cube (or the original if no fixes were - needed) metadata to ensure that it complies with the standards of its - project CMOR tables. + This method collects all the relevant fixes (including generic ones) for a + given variable and applies them. Parameters ---------- - cubes: iris.cube.CubeList + cubes: Cubes to fix. - short_name: str + short_name: Variable's short name. - project: str + project: Project of the dataset. - dataset: str + dataset: Name of the dataset. - mip: str + mip: Variable's MIP. - frequency: str, optional + frequency: Variable's data frequency, if available. - check_level: CheckLevels - Level of strictness of the checks. Set to default. - session: Session, optional + check_level: + Level of strictness of the checks. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.cmor_check_metadata`, + :func:`~esmvalcore.preprocessor.cmor_check_data`, or + :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function + will no longer perform CMOR checks. Fixes and CMOR checks have been + clearly separated in ESMValCore version 2.10.0. + session: Current session which includes configuration and directory information. - **extra_facets: dict, optional + **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. Returns ------- - iris.cube.Cube: - Fixed and checked cube. + iris.cube.CubeList + Fixed cubes. - Raises - ------ - CMORCheckError - If the checker detects errors in the metadata that it can not fix. """ + # Deprecate CMOR checks (remove in v2.12) + if check_level != CheckLevels.DEFAULT: + msg = ( + "The option `check_level` has been deprecated in ESMValCore " + "version 2.10.0 and is scheduled for removal in version 2.12.0. " + "Please use the functions " + "esmvalcore.preprocessor.cmor_check_metadata, " + "esmvalcore.preprocessor.cmor_check_data, or " + "esmvalcore.cmor.check.cmor_check instead. This function will no " + "longer perform CMOR checks. Fixes and CMOR checks have been " + "clearly separated in ESMValCore version 2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update({ @@ -152,8 +179,12 @@ def fix_metadata(cubes, mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session) - fixed_cubes = [] + session=session, + frequency=frequency) + fixed_cubes = CubeList() + + # Group cubes by input file and apply all fixes to each group element + # (i.e., each file) individually by_file = defaultdict(list) for cube in cubes: by_file[cube.attributes.get('source_file', '')].append(cube) @@ -163,94 +194,97 @@ def fix_metadata(cubes, for fix in fixes: cube_list = fix.fix_metadata(cube_list) - cube = _get_single_cube(cube_list, short_name, project, dataset) - checker = _get_cmor_checker(frequency=frequency, - table=project, - mip=mip, - short_name=short_name, - check_level=check_level, - fail_on_error=False, - automatic_fixes=True) + # The final fix is always GenericFix, whose fix_metadata method always + # returns a single cube + cube = cube_list[0] + + # Perform CMOR checks + # TODO: remove in v2.12 + checker = _get_cmor_checker( + project, + mip, + short_name, + frequency, + fail_on_error=False, + check_level=check_level, + ) cube = checker(cube).check_metadata() + cube.attributes.pop('source_file', None) fixed_cubes.append(cube) - return fixed_cubes - -def _get_single_cube(cube_list, short_name, project, dataset): - if len(cube_list) == 1: - return cube_list[0] - cube = None - for raw_cube in cube_list: - if raw_cube.var_name == short_name: - cube = raw_cube - break - if not cube: - raise ValueError( - f'More than one cube found for variable {short_name} in ' - f'{project}:{dataset} but none of their var_names match the ' - f'expected.\nFull list of cubes encountered: {cube_list}' - ) - logger.warning( - 'Found variable %s in %s:%s, but there were other present in ' - 'the file. Those extra variables are usually metadata ' - '(cell area, latitude descriptions) that was not saved ' - 'according to CF-conventions. It is possible that errors appear ' - 'further on because of this. \nFull list of cubes encountered: %s', - short_name, project, dataset, cube_list) - return cube + return fixed_cubes -def fix_data(cube, - short_name, - project, - dataset, - mip, - frequency=None, - check_level=CheckLevels.DEFAULT, - session: Optional[Session] = None, - **extra_facets): - """Fix cube data if fixes add present and check it anyway. +def fix_data( + cube: Cube, + short_name: str, + project: str, + dataset: str, + mip: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, + session: Optional[Session] = None, + **extra_facets, +) -> Cube: + """Fix cube data if fixes are required. This method assumes that metadata is already fixed and checked. - This method collects all the relevant fixes for a given variable, applies - them and checks resulting cube (or the original if no fixes were - needed) metadata to ensure that it complies with the standards of its - project CMOR tables. + This method collects all the relevant fixes (including generic ones) for a + given variable and applies them. Parameters ---------- - cube: iris.cube.Cube + cube: Cube to fix. - short_name: str + short_name: Variable's short name. - project: str + project: Project of the dataset. - dataset: str + dataset: Name of the dataset. - mip: str + mip: Variable's MIP. - frequency: str, optional + frequency: Variable's data frequency, if available. - check_level: CheckLevels - Level of strictness of the checks. Set to default. - session: Session, optional + check_level: + Level of strictness of the checks. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.cmor_check_metadata`, + :func:`~esmvalcore.preprocessor.cmor_check_data`, or + :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function + will no longer perform CMOR checks. Fixes and CMOR checks have been + clearly separated in ESMValCore version 2.10.0. + session: Current session which includes configuration and directory information. - **extra_facets: dict, optional + **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. Returns ------- - iris.cube.Cube: - Fixed and checked cube. + iris.cube.Cube + Fixed cube. - Raises - ------ - CMORCheckError - If the checker detects errors in the data that it can not fix. """ + # Deprecate CMOR checks (remove in v2.12) + if check_level != CheckLevels.DEFAULT: + msg = ( + "The option `check_level` has been deprecated in ESMValCore " + "version 2.10.0 and is scheduled for removal in version 2.12.0. " + "Please use the functions " + "esmvalcore.preprocessor.cmor_check_metadata, " + "esmvalcore.preprocessor.cmor_check_data, or " + "esmvalcore.cmor.check.cmor_check instead. This function will no " + "longer perform CMOR checks. Fixes and CMOR checks have been " + "clearly separated in ESMValCore version 2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update({ @@ -266,14 +300,20 @@ def fix_data(cube, mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session): + session=session, + frequency=frequency): cube = fix.fix_data(cube) - checker = _get_cmor_checker(frequency=frequency, - table=project, - mip=mip, - short_name=short_name, - fail_on_error=False, - automatic_fixes=True, - check_level=check_level) + + # Perform CMOR checks + # TODO: remove in v2.12 + checker = _get_cmor_checker( + project, + mip, + short_name, + frequency, + fail_on_error=False, + check_level=check_level, + ) cube = checker(cube).check_data() + return cube diff --git a/esmvalcore/cmor/fixes.py b/esmvalcore/cmor/fixes.py index e5931b0f0f..534aa3bd94 100644 --- a/esmvalcore/cmor/fixes.py +++ b/esmvalcore/cmor/fixes.py @@ -1,8 +1,15 @@ """Functions for fixing specific issues with datasets.""" -from ._fixes.shared import add_altitude_from_plev, add_plev_from_altitude +from ._fixes.shared import ( + add_altitude_from_plev, + add_plev_from_altitude, + get_next_month, + get_time_bounds, +) __all__ = [ 'add_altitude_from_plev', 'add_plev_from_altitude', + 'get_time_bounds', + 'get_next_month', ] diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 680e7289a1..ebce5431ae 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -11,8 +11,6 @@ import json import logging import os -import tempfile -import warnings from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path @@ -20,7 +18,7 @@ import yaml -from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError +from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) @@ -72,18 +70,50 @@ def _get_mips(project: str, short_name: str) -> list[str]: return mips -def get_var_info(project, mip, short_name): +def get_var_info( + project: str, + mip: str, + short_name: str, +) -> VariableInfo | None: """Get variable information. + Note + ---- + If `project=CORDEX` and the `mip` ends with 'hr', it is cropped to 'h' + since CORDEX X-hourly tables define the `mip` as ending in 'h' instead of + 'hr'. + Parameters ---------- - project : str + project: Dataset's project. - mip : str - Variable's cmor table. - short_name : str + mip: + Variable's CMOR table, i.e., MIP. + short_name: Variable's short name. + + Returns + ------- + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + + Raises + ------ + KeyError + No CMOR tables available for `project`. + """ + if project not in CMOR_TABLES: + raise KeyError( + f"No CMOR tables available for project '{project}'. The following " + f"tables are available: {', '.join(CMOR_TABLES)}." + ) + + # CORDEX X-hourly tables define the mip as ending in 'h' instead of 'hr' + if project == 'CORDEX' and mip.endswith('hr'): + mip = mip.replace('hr', 'h') + return CMOR_TABLES[project].get_variable(mip, short_name) @@ -95,34 +125,18 @@ def read_cmor_tables(cfg_developer: Optional[Path] = None) -> None: cfg_developer: Path to config-developer.yml file. - Prior to v2.8.0 `cfg_developer` was an :obj:`dict` with the contents - of config-developer.yml. This is deprecated and support will be - removed in v2.10.0. + Raises + ------ + TypeError + If `cfg_developer` is not a Path-like object """ - if isinstance(cfg_developer, dict): - warnings.warn( - "Using the `read_cmor_tables` file with a dictionary as argument " - "has been deprecated in ESMValCore version 2.8.0 and is " - "scheduled for removal in version 2.10.0. " - "Please use the path to the config-developer.yml file instead.", - ESMValCoreDeprecationWarning, - ) - with tempfile.NamedTemporaryFile( - mode='w', - encoding='utf-8', - delete=False, - ) as file: - yaml.safe_dump(cfg_developer, file) - cfg_file = Path(file.name) - else: - cfg_file = cfg_developer - if cfg_file is None: - cfg_file = Path(__file__).parents[1] / 'config-developer.yml' - mtime = cfg_file.stat().st_mtime - cmor_tables = _read_cmor_tables(cfg_file, mtime) - if isinstance(cfg_developer, dict): - # clean up the temporary file - cfg_file.unlink() + if cfg_developer is None: + cfg_developer = Path(__file__).parents[1] / 'config-developer.yml' + elif not isinstance(cfg_developer, Path): + raise TypeError("cfg_developer is not a Path-like object, got ", + cfg_developer) + mtime = cfg_developer.stat().st_mtime + cmor_tables = _read_cmor_tables(cfg_developer, mtime) CMOR_TABLES.clear() CMOR_TABLES.update(cmor_tables) @@ -144,7 +158,7 @@ def _read_cmor_tables(cfg_file: Path, mtime: float) -> dict[str, CMORTable]: cfg_developer = yaml.safe_load(file) cwd = os.path.dirname(os.path.realpath(__file__)) var_alt_names_file = os.path.join(cwd, 'variable_alt_names.yml') - with open(var_alt_names_file, 'r') as yfile: + with open(var_alt_names_file, 'r', encoding='utf-8') as yfile: alt_names = yaml.safe_load(yfile) cmor_tables: dict[str, CMORTable] = {} @@ -219,6 +233,7 @@ class InfoBase(): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, default, alt_names, strict): if alt_names is None: alt_names = "" @@ -243,28 +258,35 @@ def get_table(self, table): """ return self.tables.get(table) - def get_variable(self, table_name, short_name, derived=False): - """Search and return the variable info. + def get_variable( + self, + table_name: str, + short_name: str, + derived: Optional[bool] = False, + ) -> VariableInfo | None: + """Search and return the variable information. Parameters ---------- - table_name: str - Table name - short_name: str - Variable's short name - derived: bool, optional - Variable is derived. Info retrieval for derived variables always - look on the default tables if variable is not find in the - requested table + table_name: + Table name, i.e., the variable's MIP. + short_name: + Variable's short name. + derived: + Variable is derived. Information retrieval for derived variables + always looks in the default tables (usually, the custom tables) if + variable is not found in the requested table. Returns ------- - VariableInfo - Return the VariableInfo object for the requested variable if - found, returns None if not + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + """ alt_names_list = self._get_alt_names_list(short_name) + # First, look in requested table table = self.get_table(table_name) if table: for alt_names in alt_names_list: @@ -273,10 +295,19 @@ def get_variable(self, table_name, short_name, derived=False): except KeyError: pass + # If that didn't work, look in all tables (i.e., other MIPs) if + # cmor_strict=False var_info = self._look_in_all_tables(alt_names_list) + + # If that didn' work either, look in default table if cmor_strict=False + # or derived=True if not var_info: var_info = self._look_in_default(derived, alt_names_list, table_name) + + # If necessary, adapt frequency of variable (set it to the one from the + # requested MIP). E.g., if the user asked for table `Amon`, but the + # variable has been found in `day`, use frequency `mon`. if var_info: var_info = var_info.copy() var_info = self._update_frequency_from_mip(table_name, var_info) @@ -284,6 +315,7 @@ def get_variable(self, table_name, short_name, derived=False): return var_info def _look_in_default(self, derived, alt_names_list, table_name): + """Look for variable in default table.""" var_info = None if (not self.strict or derived): for alt_names in alt_names_list: @@ -293,6 +325,7 @@ def _look_in_default(self, derived, alt_names_list, table_name): return var_info def _look_in_all_tables(self, alt_names_list): + """Look for variable in all tables.""" var_info = None if not self.strict: for alt_names in alt_names_list: @@ -302,6 +335,7 @@ def _look_in_all_tables(self, alt_names_list): return var_info def _get_alt_names_list(self, short_name): + """Get list of alternative variable names.""" alt_names_list = [short_name] for alt_names in self.alt_names: if short_name in alt_names: @@ -312,12 +346,14 @@ def _get_alt_names_list(self, short_name): return alt_names_list def _update_frequency_from_mip(self, table_name, var_info): + """Update frequency information of var_info from table.""" mip_info = self.get_table(table_name) if mip_info: var_info.frequency = mip_info.frequency return var_info def _look_all_tables(self, alt_names): + """Look for variable in all tables.""" for table_vars in sorted(self.tables.values()): if alt_names in table_vars: return table_vars[alt_names] @@ -341,6 +377,7 @@ class CMIP6Info(InfoBase): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, cmor_tables_path, default=None, @@ -386,7 +423,7 @@ def _get_cmor_path(cmor_tables_path): 'CMOR tables not found in {}'.format(cmor_tables_path)) def _load_table(self, json_file): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: raw_data = json.loads(inf.read()) if not self._is_table(raw_data): return @@ -436,7 +473,7 @@ def _load_coordinates(self): self.coords = {} for json_file in glob.glob( os.path.join(self._cmor_folder, '*coordinate*.json')): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: table_data = json.loads(inf.read()) for coord_name in table_data['axis_entry'].keys(): coord = CoordinateInfo(coord_name) @@ -448,7 +485,7 @@ def _load_controlled_vocabulary(self): self.institutes = {} for json_file in glob.glob(os.path.join(self._cmor_folder, '*_CV.json')): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: table_data = json.loads(inf.read()) try: exps = table_data['CV']['experiment_id'] @@ -497,6 +534,7 @@ def _is_table(table_data): @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" + def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super(TableInfo, self).__init__(*args, **kwargs) @@ -522,6 +560,7 @@ class JsonInfo(object): Provides common utility methods to read json variables """ + def __init__(self): self._json_data = {} @@ -562,6 +601,7 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" + def __init__(self, table_type, short_name): """Class to read and store variable information. @@ -673,6 +713,7 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" + def __init__(self, name): """Class to read and store coordinate information. @@ -760,6 +801,7 @@ class CMIP5Info(InfoBase): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, cmor_tables_path, default=None, @@ -812,7 +854,7 @@ def _load_table(self, table_file, table_name=''): self._read_table_file(table_file, table) def _read_table_file(self, table_file, table=None): - with open(table_file) as self._current_table: + with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: key, value = self._last_line_read @@ -918,6 +960,7 @@ class CMIP3Info(CMIP5Info): If False, will look for a variable in other tables if it can not be found in the requested one """ + def _read_table_file(self, table_file, table=None): for dim in ('zlevel', ): coord = CoordinateInfo(dim) @@ -1022,7 +1065,7 @@ def get_variable(self, table, short_name, derived=False): return self.tables['custom'].get(short_name, None) def _read_table_file(self, table_file, table=None): - with open(table_file) as self._current_table: + with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: key, value = self._last_line_read diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 69dfe9658d..745626339b 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -145,11 +145,13 @@ EMAC: CORDEX: input_dir: default: '/' - spec: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}' - BADC: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{mip}/{short_name}/{version}' + spec: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{institute}-{dataset}/{rcm_version}/{mip}/{short_name}' + BADC: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{institute}-{dataset}/{rcm_version}/{mip}/{short_name}/{version}' + DKRZ: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{institute}-{dataset}/{rcm_version}/{mip}/{short_name}/{version}' ESGF: '{project.lower}/output/{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{frequency}/{short_name}/{version}' - input_file: '{short_name}_{domain}_{driver}_{exp}_{ensemble}_{dataset}_{rcm_version}_{mip}*.nc' - output_file: '{project}_{dataset}_{rcm_version}_{driver}_{domain}_{mip}_{exp}_{ensemble}_{short_name}' + SYNDA: '{domain}/{institute}/{driver}/{exp}/{ensemble}/{dataset}/{rcm_version}/{frequency}/{short_name}/{version}' + input_file: '{short_name}_{domain}_{driver}_{exp}_{ensemble}_{institute}-{dataset}_{rcm_version}_{mip}*.nc' + output_file: '{project}_{institute}_{dataset}_{rcm_version}_{driver}_{domain}_{mip}_{exp}_{ensemble}_{short_name}' cmor_type: 'CMIP5' cmor_path: 'cordex' diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 85f8f1f1c2..5dcad80e05 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -43,7 +43,7 @@ def _load_extra_facets(project, extra_facets_dir): config_file_paths = config_path.glob(f"{project.lower()}-*.yml") for config_file_path in sorted(config_file_paths): logger.debug("Loading extra facets from %s", config_file_path) - with config_file_path.open() as config_file: + with config_file_path.open(encoding='utf-8') as config_file: config_piece = yaml.safe_load(config_file) if config_piece: _deep_update(config, config_piece) diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index a0fd23e90b..0b9c596177 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -225,7 +225,7 @@ def _read_config_file(config_file): if not config_file.exists(): raise IOError(f'Config file `{config_file}` does not exist.') - with open(config_file, 'r') as file: + with open(config_file, 'r', encoding='utf-8') as file: cfg = yaml.safe_load(file) return cfg diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 736a6ba689..867370ae77 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Iterable from functools import lru_cache, partial from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import Any, Optional, Union from packaging import version @@ -23,9 +23,6 @@ InvalidConfigParameter, ) -if TYPE_CHECKING: - from ._validated_config import ValidatedConfig - logger = logging.getLogger(__name__) @@ -288,7 +285,6 @@ def validate_diagnostics( 'extra_facets_dir': validate_pathtuple, 'log_level': validate_string, 'max_parallel_tasks': validate_int_or_none, - 'offline': validate_bool, 'output_dir': validate_path, 'output_file_type': validate_string, 'profile_diagnostic': validate_bool, @@ -297,7 +293,6 @@ def validate_diagnostics( 'run_diagnostic': validate_bool, 'save_intermediary_cubes': validate_bool, 'search_esgf': validate_search_esgf, - 'use_legacy_supplementaries': validate_bool_or_none, # From CLI 'check_level': validate_check_level, @@ -340,70 +335,12 @@ def _handle_deprecation( warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) -def deprecate_offline( - validated_config: ValidatedConfig, - value: Any, - validated_value: Any, -) -> None: - """Deprecate ``offline`` option. - - Parameters - ---------- - validated_config: ValidatedConfig - ``ValidatedConfig`` instance which will be modified in place. - value: Any - Raw input value for ``offline`` option. - validated_value: Any - Validated value for ``offline`` option. - - """ - option = 'offline' - deprecated_version = '2.8.0' - remove_version = '2.10.0' - more_info = ( - " Please use the options `search_esgf=never` (for `offline=True`) or " - "`search_esgf=when_missing` (for `offline=False`) instead. These are " - "exact replacements." - ) - _handle_deprecation(option, deprecated_version, remove_version, more_info) - if validated_value: - validated_config['search_esgf'] = 'never' - else: - validated_config['search_esgf'] = 'when_missing' - - -def deprecate_use_legacy_supplementaries( - validated_config: ValidatedConfig, - value: Any, - validated_value: Any, -) -> None: - """Deprecate ``use_legacy_supplementaries`` option. - - Parameters - ---------- - validated_config: ValidatedConfig - ``ValidatedConfig`` instance which will be modified in place. - value: Any - Raw input value for ``use_legacy_supplementaries`` option. - validated_value: Any - Validated value for ``use_legacy_supplementaries`` option. - - """ - option = 'use_legacy_supplementaries' - deprecated_version = '2.8.0' - remove_version = '2.10.0' - more_info = '' - _handle_deprecation(option, deprecated_version, remove_version, more_info) - - -_deprecators: dict[str, Callable] = { - 'offline': deprecate_offline, - 'use_legacy_supplementaries': deprecate_use_legacy_supplementaries, -} +# Example usage: see removed files in +# https://github.com/ESMValGroup/ESMValCore/pull/2213 +_deprecators: dict[str, Callable] = {} # Default values for deprecated options -_deprecated_options_defaults: dict[str, Any] = { - 'offline': True, - 'use_legacy_supplementaries': None, -} +# Example usage: see removed files in +# https://github.com/ESMValGroup/ESMValCore/pull/2213 +_deprecated_options_defaults: dict[str, Any] = {} diff --git a/esmvalcore/config/_diagnostics.py b/esmvalcore/config/_diagnostics.py index e54dba87a6..c8f0869c9e 100644 --- a/esmvalcore/config/_diagnostics.py +++ b/esmvalcore/config/_diagnostics.py @@ -83,7 +83,7 @@ def from_file(cls, filename: str): """Load the reference tags used for provenance recording.""" if os.path.exists(filename): logger.debug("Loading tags from %s", filename) - with open(filename) as file: + with open(filename, 'r', encoding='utf-8') as file: tags = cls(yaml.safe_load(file)) tags.source_file = filename return tags diff --git a/esmvalcore/config/_esgf_pyclient.py b/esmvalcore/config/_esgf_pyclient.py index 7e80cc4892..1d473068e6 100644 --- a/esmvalcore/config/_esgf_pyclient.py +++ b/esmvalcore/config/_esgf_pyclient.py @@ -110,7 +110,7 @@ def read_config_file(): if mode & stat.S_IRWXG or mode & stat.S_IRWXO: logger.warning("Correcting unsafe permissions on %s", CONFIG_FILE) os.chmod(CONFIG_FILE, stat.S_IRUSR | stat.S_IWUSR) - with CONFIG_FILE.open() as file: + with CONFIG_FILE.open(encoding='utf-8') as file: cfg = yaml.safe_load(file) else: logger.info( diff --git a/esmvalcore/config/_logging.py b/esmvalcore/config/_logging.py index 674fad953d..3763c3b688 100644 --- a/esmvalcore/config/_logging.py +++ b/esmvalcore/config/_logging.py @@ -87,7 +87,7 @@ def configure_logging( cfg_file = Path(cfg_file).absolute() - with open(cfg_file) as file_handler: + with open(cfg_file, 'r', encoding='utf-8') as file_handler: cfg = yaml.safe_load(file_handler) if output_dir is None: diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 35c8588404..d4bd665aa6 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -677,19 +677,15 @@ def load(self) -> Cube: iris.cube.Cube An :mod:`iris` cube with the data corresponding the the dataset. """ - return self._load_with_callback(callback='default') - - def _load_with_callback(self, callback): - # TODO: Remove the callback argument for v2.10.0. input_files = list(self.files) for supplementary_dataset in self.supplementaries: input_files.extend(supplementary_dataset.files) esgf.download(input_files, self.session['download_dir']) - cube = self._load(callback) + cube = self._load() supplementary_cubes = [] for supplementary_dataset in self.supplementaries: - supplementary_cube = supplementary_dataset._load(callback) + supplementary_cube = supplementary_dataset._load() supplementary_cubes.append(supplementary_cube) output_file = _get_output_file(self.facets, self.session.preproc_dir) @@ -704,7 +700,7 @@ def _load_with_callback(self, callback): return cubes[0] - def _load(self, callback) -> Cube: + def _load(self) -> Cube: """Load self.files into an iris cube and return it.""" if not self.files: lines = [ @@ -731,7 +727,6 @@ def _load(self, callback) -> Cube: **self.facets, } settings['load'] = { - 'callback': callback, 'ignore_warnings': get_ignored_warnings( self.facets['project'], 'load' ), @@ -741,7 +736,9 @@ def _load(self, callback) -> Cube: 'session': self.session, **self.facets, } - settings['concatenate'] = {} + settings['concatenate'] = { + 'check_level': self.session['check_level'] + } settings['cmor_check_metadata'] = { 'check_level': self.session['check_level'], 'cmor_table': self.facets['project'], @@ -862,7 +859,7 @@ def _update_timerange(self): dataset.facets.pop('timerange') dataset.supplementaries = [] check.data_availability(dataset) - intervals = [_get_start_end_date(f.name) for f in dataset.files] + intervals = [_get_start_end_date(f) for f in dataset.files] min_date = min(interval[0] for interval in intervals) max_date = max(interval[1] for interval in intervals) diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index 7efe389293..551aacceed 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -52,7 +52,7 @@ def compute_speed(size, duration): def load_speeds(): """Load average download speeds from HOSTS_FILE.""" try: - content = HOSTS_FILE.read_text() + content = HOSTS_FILE.read_text(encoding='utf-8') except FileNotFoundError: content = '{}' speeds = yaml.safe_load(content) @@ -94,7 +94,7 @@ def atomic_write(filename): filename.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile(prefix=f"{filename}.") as file: tmp_file = file.name - with open(tmp_file, 'w') as file: + with open(tmp_file, 'w', encoding='utf-8') as file: yield file shutil.move(tmp_file, filename) diff --git a/esmvalcore/esgf/_search.py b/esmvalcore/esgf/_search.py index 87d4cdf095..62882a5345 100644 --- a/esmvalcore/esgf/_search.py +++ b/esmvalcore/esgf/_search.py @@ -168,7 +168,7 @@ def select_by_time(files, timerange): for file in files: start_date, end_date = _parse_period(timerange) try: - start, end = _get_start_end_date(file.name) + start, end = _get_start_end_date(file) except ValueError: # If start and end year cannot be read from the filename # just select everything. diff --git a/esmvalcore/experimental/recipe.py b/esmvalcore/experimental/recipe.py index 5839d72df8..18e520324b 100644 --- a/esmvalcore/experimental/recipe.py +++ b/esmvalcore/experimental/recipe.py @@ -70,7 +70,8 @@ def name(self): def data(self) -> dict: """Return dictionary representation of the recipe.""" if self._data is None: - self._data = yaml.safe_load(open(self.path, 'r')) + with open(self.path, 'r', encoding='utf-8') as yaml_file: + self._data = yaml.safe_load(yaml_file) return self._data def _load(self, session: Session) -> RecipeEngine: diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py index 0e00ea3667..cb0fd32f95 100644 --- a/esmvalcore/experimental/recipe_info.py +++ b/esmvalcore/experimental/recipe_info.py @@ -72,7 +72,7 @@ def _repr_html_(self) -> str: @classmethod def from_yaml(cls, path: str): """Return instance of 'RecipeInfo' from a recipe in yaml format.""" - data = yaml.safe_load(open(path, 'r')) + data = yaml.safe_load(Path(path).read_text(encoding='utf-8')) return cls(data, filename=path) @property diff --git a/esmvalcore/experimental/recipe_output.py b/esmvalcore/experimental/recipe_output.py index bcb38c019a..9765d15c77 100644 --- a/esmvalcore/experimental/recipe_output.py +++ b/esmvalcore/experimental/recipe_output.py @@ -2,7 +2,7 @@ import base64 import logging import os.path -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Optional, Tuple, Type @@ -123,6 +123,13 @@ class RecipeOutput(Mapping): The session used to run the recipe. """ + FILTER_ATTRS: list = [ + "realms", + "plot_type", # Used by several diagnostics + "plot_types", + "long_names", + ] + def __init__(self, task_output: dict, session=None, info=None): self._raw_task_output = task_output self._task_output = {} @@ -141,6 +148,7 @@ def __init__(self, task_output: dict, session=None, info=None): diagnostics[name].append(task) # Create diagnostic output + filters: dict = {} for name, tasks in diagnostics.items(): diagnostic_info = info.data['diagnostics'][name] self.diagnostics[name] = DiagnosticOutput( @@ -150,6 +158,36 @@ def __init__(self, task_output: dict, session=None, info=None): description=diagnostic_info.get('description'), ) + # Add data to filters + for task in tasks: + for file in task.files: + RecipeOutput._add_to_filters(filters, file.attributes) + + # Sort at the end because sets are unordered + self.filters = RecipeOutput._sort_filters(filters) + + @classmethod + def _add_to_filters(cls, filters, attributes): + """Add valid values to the HTML output filters.""" + for attr in RecipeOutput.FILTER_ATTRS: + if attr not in attributes: + continue + values = attributes[attr] + # `set()` to avoid duplicates + attr_list = filters.get(attr, set()) + if (isinstance(values, str) or not isinstance(values, Sequence)): + attr_list.add(values) + else: + attr_list.update(values) + filters[attr] = attr_list + + @classmethod + def _sort_filters(cls, filters): + """Sort the HTML output filters.""" + for _filter, _attrs in filters.items(): + filters[_filter] = sorted(_attrs) + return filters + def __repr__(self): """Return canonical string representation.""" string = '\n'.join(repr(item) for item in self._task_output.values()) @@ -200,7 +238,7 @@ def write_html(self): template = get_template('recipe_output_page.j2') html_dump = self.render(template=template) - with open(filename, 'w') as file: + with open(filename, 'w', encoding='utf-8') as file: file.write(html_dump) logger.info("Wrote recipe output to:\nfile://%s", filename) @@ -218,6 +256,7 @@ def render(self, template=None): diagnostics=self.diagnostics.values(), session=self.session, info=self.info, + filters=self.filters, relpath=os.path.relpath, ) @@ -225,11 +264,11 @@ def render(self, template=None): def read_main_log(self) -> str: """Read log file.""" - return self.session.main_log.read_text() + return self.session.main_log.read_text(encoding='utf-8') def read_main_log_debug(self) -> str: """Read debug log file.""" - return self.session.main_log_debug.read_text() + return self.session.main_log_debug.read_text(encoding='utf-8') class OutputFile(): diff --git a/esmvalcore/experimental/templates/RecipeOutput.j2 b/esmvalcore/experimental/templates/RecipeOutput.j2 index c502c39a33..cde84362f9 100644 --- a/esmvalcore/experimental/templates/RecipeOutput.j2 +++ b/esmvalcore/experimental/templates/RecipeOutput.j2 @@ -1,12 +1,72 @@ + + + + +
{% for diagnostic in diagnostics %} -

{{ diagnostic.title }}

-

{{ diagnostic.description }}

+
+

{{ diagnostic.title }}

+

{{ diagnostic.description }}

- {% for task in diagnostic.task_output %} + {% set diagnostic_loop = loop %} + {% for task in diagnostic.task_output %} - {% include 'TaskOutput.j2' %} + {% include 'TaskOutput.j2' %} - {% endfor %} + {% endfor %} +
{% endfor %} +
diff --git a/esmvalcore/experimental/templates/TaskOutput.j2 b/esmvalcore/experimental/templates/TaskOutput.j2 index 48d259ea7c..43753418b2 100644 --- a/esmvalcore/experimental/templates/TaskOutput.j2 +++ b/esmvalcore/experimental/templates/TaskOutput.j2 @@ -2,11 +2,23 @@ {% for file in task.image_files %} -
+
+
- {{ file.caption }} + {{ file.caption }} -
+
{{ file.caption }}

@@ -16,21 +28,28 @@ provenance
+
{% endfor %} -

Data files

+{% if task.data_files|length > 0 %} +

Data files

- + + +{% endif %} diff --git a/esmvalcore/experimental/templates/head.j2 b/esmvalcore/experimental/templates/head.j2 index e6ec19d5ee..0306620f8a 100644 --- a/esmvalcore/experimental/templates/head.j2 +++ b/esmvalcore/experimental/templates/head.j2 @@ -2,6 +2,9 @@ {{ title }} + + +