From 9b9a12526d9afdc87a5dd9e6904efe37acb629ac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:53:43 +0100 Subject: [PATCH 01/17] [pre-commit.ci] pre-commit autoupdate (#2581) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f2f375f42..dbc9b3117b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.7.3" + rev: "v0.7.4" hooks: - id: ruff args: [--fix] From 11e6aa877c35639586e1eb27b5751c096b8aa68d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 13:39:02 +0000 Subject: [PATCH 02/17] [Condalock] Update Linux condalock file (#2583) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 220 +++++++++++++++++++++++--------------------- 1 file changed, 116 insertions(+), 104 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 3a0d7b7b16..260725b5df 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: 090d7bbcb310d32c80fa50bdedeee5d4a266ff46613d62244bae0b69ede2d43d +# input_hash: efb0d40da21331f3809f3aac8456fd160657e1a1f90bfc9642cbde579ae0920e @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.8.30-hbcca054_0.conda#c27d1c142233b5bc9ca570c6e2e0c244 @@ -10,15 +10,17 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77 https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda#49023d73832ef61042f6a237cb2687e7 https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.5-ha770c72_0.conda#2889e6b9c666c3a564ab90cedc5832fd https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda#d8d7293c5b37f39b2ac32940621c6592 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.12-5_cp312.conda#0424ae29b104430108f5218a66db7260 +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.13-5_cp313.conda#381bbd2a92c863f640a55b6ff3c35161 https://conda.anaconda.org/conda-forge/noarch/tzdata-2024b-hc8b5060_0.conda#8ac3367aafb1cc0a068483c580af8015 https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda#048b02e3962f066da18efe3a21b77672 +https://conda.anaconda.org/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda#434ca7e50e40f4918ab701e3facd59a0 https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h77fa898_1.conda#cc3573974587f12dda90d96e3e55a702 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab +https://conda.anaconda.org/conda-forge/linux-64/libegl-1.7.0-ha4b6fd6_2.conda#c151d5eb730e9b7480e6d48c0fc44048 https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h77fa898_1.conda#3cb76c3f10d3bc7f1105b2fc9db984df -https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.10.0-hb9d3cd8_0.conda#f6495bc3a19a4400d3407052d22bef13 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.10.3-hb9d3cd8_0.conda#ff3653946d34a6a6ba10babb139d96ef https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.3-heb4867d_0.conda#09a6c610d002e54e18353c06ef61a253 https://conda.anaconda.org/conda-forge/linux-64/json-c-0.18-h6688a6e_0.conda#38f5dbc9ac808e31c00650f7be1db93f https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb9d3cd8_2.conda#41b599ed2b02abcfdd84302bff174b23 @@ -28,20 +30,20 @@ https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_1.cond https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-14.2.0-hd5240d6_1.conda#9822b874ea29af082e5d36098d25427d https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-hc0a3c3a_1.conda#234a5554c53625688d51062645337328 https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8 -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.2-hb9d3cd8_0.conda#4d638782050ab6faa27275bed57e9b4e +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.4.0-hb9d3cd8_0.conda#23cc74f77eb99315c0360ec3533147a9 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda#b3c17d95b5a10c6e64a21fa17573e70e https://conda.anaconda.org/conda-forge/linux-64/tzcode-2024b-hb9d3cd8_0.conda#db124840386e1f842f93372897d1b857 https://conda.anaconda.org/conda-forge/linux-64/xorg-libice-1.1.1-hb9d3cd8_1.conda#19608a9656912805b2b9a2f6bd257b04 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxau-1.0.11-hb9d3cd8_1.conda#77cbc488235ebbaab2b6e912d3934bae https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb9d3cd8_0.conda#8035c64cb77ed555e3f150b7b3972480 -https://conda.anaconda.org/conda-forge/linux-64/xorg-xextproto-7.3.0-hb9d3cd8_1004.conda#bc4cd53a083b6720d61a1519a1900878 https://conda.anaconda.org/conda-forge/linux-64/xorg-xorgproto-2024.1-hb9d3cd8_1.conda#7c21106b851ec72c037b162c216d8f05 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.8.0-he70792b_1.conda#9b81a9d9395fb2abd60984fcfe7eb01a -https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.3.0-hba2fe39_1.conda#c6133966058e553727f0afe21ab38cd2 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.2.1-hba2fe39_0.conda#f0b3524e47ed870bb8304a8d6fa67c7f -https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.2.0-hba2fe39_1.conda#ac8d58d81bdcefa5bce4e883c6b88c42 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.8.0-hecf86a2_2.conda#c54459d686ad9d0502823cacff7e8423 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.3.0-hf42f96a_2.conda#257f4ae92fe11bd8436315c86468c39b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.2.1-hf42f96a_1.conda#bbdd20fb1994a9f0ba98078fcb6c12ab +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.2.2-hf42f96a_1.conda#d908d43d87429be24edfb20e96543c20 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553 https://conda.anaconda.org/conda-forge/linux-64/capnproto-1.0.2-h766bdaa_3.conda#7ea5f8afe8041beee8bad281dee62414 +https://conda.anaconda.org/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda#418c6ca5929a611cbd69204907a83995 https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2#ac7bc6a654f8f41b352b38f4051135f8 https://conda.anaconda.org/conda-forge/linux-64/geos-3.13.0-h5888daf_0.conda#40b4ab956c90390e407bb177f8a58bab https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda#d411fc29e338efb48c5fd4576d71d881 @@ -56,8 +58,10 @@ https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/libgfortran-14.2.0-h69a702a_1.conda#f1fd30127802683586f768875127a987 https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-hd590300_2.conda#d66573916ffcf376178462f1b61c941e https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-3.0.0-hd590300_1.conda#ea25936bb4080d843790b586850f82b8 +https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-h4bc722e_0.conda#aeb98fdeb2e8f25d43ef71fbacbeec80 https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda#30fd6e37fe21f86f4bd26d6ee73eeec7 https://conda.anaconda.org/conda-forge/linux-64/libntlm-1.4-h7f98852_1002.tar.bz2#e728e874159b042d92b90238a3cb0dc2 +https://conda.anaconda.org/conda-forge/linux-64/libpciaccess-0.18-hd590300_0.conda#48f4330bfcd959c3cfb704d424903c82 https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.44-hadc24fc_0.conda#f4cc49d7aa68316213e4b12be35308d1 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.20-h4ab18f5_0.conda#a587892d3c13b6621a6091be690dbca2 https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.47.0-hadc24fc_1.conda#b6f02b52a174e612e89548f4663ce56a @@ -68,17 +72,19 @@ https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.4.0-hd590300_0.conda#b26e8aa824079e1be0294e7152ca4559 https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda#92ed62436b625154323d40d5f2f11dd7 -https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda#5aa797f8787fe7a17d1b0821485b5adc https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-hd590300_1001.conda#ec7398d21e2651e0dcb0044d03b9a339 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda#70caf8bb6cf39a0b6b7efc885f51c0fe https://conda.anaconda.org/conda-forge/linux-64/nspr-4.36-h5888daf_0.conda#de9cd5bca9e4918527b9b72b6e2e1409 -https://conda.anaconda.org/conda-forge/linux-64/s2n-1.5.7-hd3e8b83_0.conda#b0de6ca344b9255f4adb98e419e130ad +https://conda.anaconda.org/conda-forge/linux-64/rav1e-0.6.6-he8a937b_2.conda#77d9955b4abddb811cb8ab1aa7d743e4 +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.5.9-h0fd0ee4_0.conda#f472432f3753c5ca763d2497e2ea30bf +https://conda.anaconda.org/conda-forge/linux-64/svt-av1-2.3.0-h5888daf_0.conda#355898d24394b2af353eb96358db9fdd https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.2-hd590300_0.conda#f08fb5c89edfc4aadee1c81d4cfb1fa1 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda#c9f075ab2f33b3bbee9e62d4ad0a6cd8 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.15.1-h2a50c78_1.conda#67dfecff4c4253cfe33c3c8e428f1767 +https://conda.anaconda.org/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda#346722a0be40f6edc53f12640d301338 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.15.2-hdeadb07_2.conda#461a1eaa075fd391add91bcffc9de0c1 https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hb9d3cd8_2.conda#c63b5e52939e795ba8d26e35d767a843 https://conda.anaconda.org/conda-forge/linux-64/fmt-11.0.2-h434a139_0.conda#995f7e13598497691c1dc476d889bc04 https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda#9ae35c3d96db2c94ce0cef86efdfa2cb @@ -89,6 +95,8 @@ https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda#8b1893 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f https://conda.anaconda.org/conda-forge/linux-64/libaec-1.1.3-h59595ed_0.conda#5e97e271911b8b2001a8b71860c32faa https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 +https://conda.anaconda.org/conda-forge/linux-64/libde265-1.0.15-h00ab1b0_0.conda#407fee7a5d7ab2dca12c9ca7f62310ad +https://conda.anaconda.org/conda-forge/linux-64/libdrm-2.4.123-hb9d3cd8_0.conda#ee605e794bdc14e2b7f84c4faa0d8c2c https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-14.2.0-h69a702a_1.conda#0a7f4cd238267c88e5d69f7826a407eb https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.64.0-h161d5f1_0.conda#19e57602824042dfd0446292ef90488b @@ -107,23 +115,26 @@ https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47 https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.1-ha2e4443_0.conda#6b7dcc7349efd123d493d2dbe85a045f https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_3.conda#6bb8deb138f87c9d48320ac21b87e7a1 https://conda.anaconda.org/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda#d71d3a66528853c0a1ac2c02d79a0284 +https://conda.anaconda.org/conda-forge/linux-64/x265-3.5-h924138e_3.tar.bz2#e7f6ed84d4623d52ee581325c1587a6b https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-he73a12e_1.conda#05a8ea5f446de33006171a7afe6ae857 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.10-h4f16b4b_0.conda#0b666058a179b744a622d0a4a0c56353 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.6-ha6fb4c9_0.conda#4d056880988120e29d75bfff282e0f45 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.5.0-h127f702_4.conda#81d250bca40c7907ece5f5dd00de51d0 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.9.0-h8a7d7e2_5.conda#c40bb8a9f3936ebeea804e97c5adf179 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.5.0-h1ffe551_7.conda#7cce4dfab184f4bbdfc160789251b3c5 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.9.1-hab05fe4_2.conda#fb409f7053fa3dbbdf6eb41045a87795 https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.6-hef167b5_0.conda#54fe76ab3d0189acaef95156874db7f9 https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hb9d3cd8_2.conda#98514fe74548d768907ce7a13f680e8f https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.15.0-h7e30c49_1.conda#8f5b0b297b59e1ac160ad4beec99dbee https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.3-h659f571_0.conda#3f43953b7d3fb3aaa1d0d0723d91e368 +https://conda.anaconda.org/conda-forge/linux-64/libavif16-1.1.1-h1909e37_2.conda#21e468ed3786ebcb2124b123aa2484b7 https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-25_linux64_openblas.conda#8ea26d42ca88ec5258802715fe1ee10b https://conda.anaconda.org/conda-forge/linux-64/libglib-2.82.2-h2ff4ddf_0.conda#13e8e54035ddd2b91875ba399f0f7c04 +https://conda.anaconda.org/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda#c8013e438185f33b13814c5c488acd5c https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-hf539b9f_1021.conda#e8c7620cc49de0c6a2349b6dd6e39beb https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.7.0-he137b08_1.conda#63872517c98aa305da58a757c443698e -https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.4-hb346dea_2.conda#69b90b70c434b916abf5a1d5ee5d55fb +https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.5-hb346dea_0.conda#c81a9f1118541aaa418ccb22190c817e https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.7-h401b404_0.conda#4474532a312b2245c5c77f1176989b46 -https://conda.anaconda.org/conda-forge/linux-64/orc-2.0.2-he039a57_2.conda#5e7bb9779cc5c200e63475eb2538d382 -https://conda.anaconda.org/conda-forge/linux-64/python-3.12.7-hc5c86c4_0_cpython.conda#0515111a9cdf69f83278f7c197db9807 +https://conda.anaconda.org/conda-forge/linux-64/orc-2.0.3-he039a57_0.conda#052499acd6d6b79952197a13b23e2600 +https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0-h9ebbce0_100_cp313.conda#08e9aef080f33daeb192b2ddc7e4721f https://conda.anaconda.org/conda-forge/linux-64/re2-2024.07.02-h77b4e00_1.conda#01093ff37c1b5e6bf9f17c0116747d11 https://conda.anaconda.org/conda-forge/linux-64/spdlog-1.14.1-hed91bc2_1.conda#909188c8979846bac8e586908cf1ca6a https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.47.0-h9eae976_1.conda#53abf1ef70b9ae213b22caa5350f97a9 @@ -132,12 +143,12 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.6-hb9d3cd8_0.co https://conda.anaconda.org/conda-forge/linux-64/xorg-libxfixes-6.0.1-hb9d3cd8_0.conda#4bdb303603e9821baf5fe5fdff1dc8f8 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hb9d3cd8_1.conda#a7a49a8b85122b49214798321e2e96b4 https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_0.conda#7d78a232029458d0077ede6cda30ed0c -https://conda.anaconda.org/conda-forge/linux-64/astroid-3.3.5-py312h7900ff3_0.conda#e1ed4d572a4a16b97368ab00fd646487 +https://conda.anaconda.org/conda-forge/linux-64/astroid-3.3.5-py313h78bf25f_0.conda#5266713116fd050a2e4d3c2de84e9fd5 https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-h04ea711_2.conda#f730d54ba9cd543666d7220c9f7ed563 https://conda.anaconda.org/conda-forge/noarch/attrs-24.2.0-pyh71513ae_0.conda#6732fa52eb8e66e5afeb32db8701a791 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.8.0-hcd8ed7f_7.conda#b8725d87357c876b0458dee554c39b28 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.11.0-hd25e75f_5.conda#277dae7174fd2bc81c01411039ac2173 -https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py312h2ec8cdc_2.conda#b0b867af6fc74b2a0aa206da29c0f3cf +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.8.0-hb88c0a9_10.conda#409b7ee6d3473cc62bda7280f6ac20d0 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.11.0-h7bd072d_8.conda#0e9d67838114c0dbd267a9311268b331 +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py313h46c70d0_2.conda#f6bb3742e17a4af0dc3c8ca942683ef6 https://conda.anaconda.org/conda-forge/linux-64/cairo-1.18.0-hebfffa5_3.conda#fceaedf1cdbcb02df9699a0d9b005292 https://conda.anaconda.org/conda-forge/noarch/certifi-2024.8.30-pyhd8ed1ab_0.conda#12f7d00853807b0531775e9be891cb11 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c @@ -147,7 +158,7 @@ https://conda.anaconda.org/conda-forge/noarch/cloudpickle-3.1.0-pyhd8ed1ab_1.con https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2#3faab06a954c2a04039983f2c4a50d99 https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_0.conda#5cd86562580f274031ede6aa6aa24441 https://conda.anaconda.org/conda-forge/linux-64/cyrus-sasl-2.1.27-h54b06d7_7.conda#dce22f70b4e5a407ce88f2be046f4ceb -https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.11-py312h8fd2918_3.conda#21e433caf1bb1e4c95832f8bb731d64c +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.11-py313hc66aa0d_3.conda#1778443eb12b2da98428fa69152a2a2e https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.bz2#43afe5ab04e35e17ba28649471dd7364 https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2#961b3a227b437d82ad7054484cfa71b2 https://conda.anaconda.org/conda-forge/noarch/dill-0.3.9-pyhd8ed1ab_0.conda#27faec84454995f6774786c7e5833cd6 @@ -172,34 +183,36 @@ https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.b https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda#f800d2da156d08e289b14e87e43c1ae5 https://conda.anaconda.org/conda-forge/noarch/isodate-0.7.2-pyhd8ed1ab_0.conda#d68d25aca67d1a06bf6f5b43aea9430d https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.2.0-pyhd8ed1ab_0.conda#ff7ca04134ee8dde1d7cf491a78ef7c7 -https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py312h68727a3_0.conda#444266743652a4f1538145e9362f6d3b +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.7-py313h33d0bda_0.conda#9862d13a5e466273d5a4738cffcb8d6c https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.16-hb7c19ff_0.conda#51bb7010fc86f70eee639b4bb7a894f5 https://conda.anaconda.org/conda-forge/noarch/legacy-cgi-2.6.1-pyh5b84bb0_3.conda#f258b7f54b5d9ddd02441f10c4dca2ac https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.4-hfca40fe_0.conda#32ddb97f897740641d8d46a829ce1704 https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-25_linux64_openblas.conda#5dbd1b0fc0d01ec5e0e1fbe667281a11 https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.10.1-hbbe4b11_0.conda#6e801c50a40301f6978c53976917b277 https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-hd3e95f3_10.conda#30ee3a29c84cf7b842a8c5828c4b7c13 -https://conda.anaconda.org/conda-forge/linux-64/libglu-9.0.0-ha6d2627_1004.conda#df069bea331c8486ac21814969301c1f +https://conda.anaconda.org/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda#928b8be80851f5d8ffb016f9c81dae7a https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.67.1-hc2c308b_0.conda#4606a4647bfe857e3cfe21ca12ac3afb +https://conda.anaconda.org/conda-forge/linux-64/libheif-1.18.2-gpl_hffcb242_100.conda#76ac2c07b62d45c192940f010eea11fa https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-25_linux64_openblas.conda#4dc03a53fc69371a6158d0ed37214cd3 https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.39-h76b75d6_0.conda#e71f31f8cfb0a91439f2086fc8aa0461 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 -https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.3-py312hb3f7f12_1.conda#b99d90ef4e77acdab74828f79705a919 -https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py312h178313f_0.conda#a755704ea0e2503f8c227d84829a8e81 +https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.3-py313h010b13d_1.conda#08a6b03e282748f599c55bbbdbd722fa +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.2-py313h8060acc_0.conda#ab825f8b676368beb91350c6a2da6e11 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.7.0-pyhd8ed1ab_0.tar.bz2#34fc335fc50eef0b5ea708f2b5f54e0c https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.2-pyhd8ed1ab_0.conda#5cbee699846772cc939bef23a0d524ed -https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.1.0-py312h68727a3_0.conda#5c9b020a3f86799cdc6115e55df06146 +https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.1.0-py313h33d0bda_0.conda#7f907b1065247efa419bb70d3a3341b5 https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 -https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyhd8ed1ab_1.conda#1d4c088869f206413c59acdd309908b7 +https://conda.anaconda.org/conda-forge/noarch/networkx-3.4.2-pyh267e887_2.conda#fd40bf7f7f4bc4b647dc8512053d9873 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.2-h488ebb8_0.conda#7f2e286780f072ed750df46dc2631138 -https://conda.anaconda.org/conda-forge/noarch/packaging-24.1-pyhd8ed1ab_0.conda#cbe1bb1f21567018ce595d9c2be0f0db +https://conda.anaconda.org/conda-forge/noarch/packaging-24.2-pyhff2d567_1.conda#8508b703977f4c4ada34d657d051972c https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9 https://conda.anaconda.org/conda-forge/noarch/parso-0.8.4-pyhd8ed1ab_0.conda#81534b420deb77da8833f2289b8d47ac https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-py_1003.tar.bz2#415f0ebb6198cc2801c73438a9fb5761 +https://conda.anaconda.org/conda-forge/noarch/pip-24.3.1-pyh145f28c_0.conda#ca3afe2d7b893a8c8cdf489d30a2b1a3 https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_1.conda#405678b942f2481cecdb3e010f4925d9 https://conda.anaconda.org/conda-forge/noarch/platformdirs-4.3.6-pyhd8ed1ab_0.conda#fd8f2b18b65bbf62e8f653100690c8d2 https://conda.anaconda.org/conda-forge/noarch/pluggy-1.5.0-pyhd8ed1ab_0.conda#d3483c8fc2dc2cc3f5cf43e26d60cabf -https://conda.anaconda.org/conda-forge/linux-64/psutil-6.1.0-py312h66e93f0_0.conda#0524eb91d3d78d76d671c6e3cd7cee82 +https://conda.anaconda.org/conda-forge/linux-64/psutil-6.1.0-py313h536fd9c_0.conda#b50a00ebd2fda55306b8a095363ce27f https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd3deb0d_0.tar.bz2#359eeb6536da0e687af562ed265ec263 https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.3-pyhd8ed1ab_0.conda#0f051f09d992e0d08941706ad519ee0e https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyhd8ed1ab_0.conda#844d9eb3b43095b031874477f7d70088 @@ -209,11 +222,11 @@ https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#9 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.20.0-pyhd8ed1ab_0.conda#b98d2018c01ce9980c03ee2850690fab https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2024.2-pyhd8ed1ab_0.conda#986287f89929b2d629bd6ef6497dc307 -https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.5.0-py312h66e93f0_1.conda#39aed2afe4d0cf76ab3d6b09eecdbea7 +https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.5.0-py313h536fd9c_1.conda#5c44ffac1f568dc8b4afb09a3e825d49 https://conda.anaconda.org/conda-forge/noarch/pytz-2024.1-pyhd8ed1ab_0.conda#3eeeeb9e4827ace8c0c1419c85d590ad -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.2-py312h66e93f0_1.conda#549e5930e768548a89c23f595dac5a95 -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.21.0-py312h12e396e_0.conda#37f4ad7cb4214c799f32e5f411c6c69f -https://conda.anaconda.org/conda-forge/noarch/setuptools-75.3.0-pyhd8ed1ab_0.conda#2ce9825396daf72baabaade36cee16da +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.2-py313h536fd9c_1.conda#3789f360de131c345e96fbfc955ca80b +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.21.0-py313h920b4c0_0.conda#4877cdeada83444c17df70a77a243da9 +https://conda.anaconda.org/conda-forge/noarch/setuptools-75.5.0-pyhff2d567_0.conda#ade63405adb52eeff89d506cd55908c0 https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2#4d22a9315e78c6827f806065957d566e https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_0.tar.bz2#6d6552722448103793743dabfbda532d @@ -222,37 +235,37 @@ https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed https://conda.anaconda.org/conda-forge/noarch/tblib-3.0.0-pyhd8ed1ab_0.conda#04eedddeb68ad39871c8127dd1c21f4f https://conda.anaconda.org/conda-forge/noarch/termcolor-2.5.0-pyhd8ed1ab_0.conda#29a5d22565b850099cd9959862d1b154 https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 -https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.2-pyhd8ed1ab_0.conda#e977934e00b355ff55ed154904044727 +https://conda.anaconda.org/conda-forge/noarch/tomli-2.1.0-pyhff2d567_0.conda#3fa1089b4722df3a900135925f4519d9 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.13.2-pyha770c72_0.conda#0062a5f3347733f67b0f33ca48cc21dd https://conda.anaconda.org/conda-forge/noarch/toolz-1.0.0-pyhd8ed1ab_0.conda#34feccdd4177f2d3d53c73fc44fd9a37 -https://conda.anaconda.org/conda-forge/linux-64/tornado-6.4.1-py312h66e93f0_1.conda#af648b62462794649066366af4ecd5b0 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.4.1-py313h536fd9c_1.conda#70b5b6dfd7d1760cd59849e2271d937b https://conda.anaconda.org/conda-forge/noarch/traitlets-5.14.3-pyhd8ed1ab_0.conda#3df84416a021220d8b5700c613af2dc5 https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.12.2-pyha770c72_0.conda#ebe6952715e1d5eb567eeebf25250fa7 -https://conda.anaconda.org/conda-forge/linux-64/ujson-5.10.0-py312h2ec8cdc_1.conda#96226f62dddc63226472b7477d783967 -https://conda.anaconda.org/conda-forge/linux-64/unicodedata2-15.1.0-py312h66e93f0_1.conda#588486a61153f94c7c13816f7069e440 +https://conda.anaconda.org/conda-forge/linux-64/ujson-5.10.0-py313h46c70d0_1.conda#7f4872b663aafde0f532543488656f5d https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.13-pyhd8ed1ab_0.conda#68f0738df502a14213624b288c60c9ad https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 -https://conda.anaconda.org/conda-forge/noarch/wheel-0.45.0-pyhd8ed1ab_0.conda#f9751d7c71df27b2d29f5cab3378982e +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxdamage-1.1.6-hb9d3cd8_0.conda#b5fcc7172d22516e1f965490e65e33a4 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda#17dcc85db3c7886650b8908b183d6876 +https://conda.anaconda.org/conda-forge/linux-64/xorg-libxxf86vm-1.1.5-hb9d3cd8_4.conda#7da9007c0582712c4bad4131f89c8372 https://conda.anaconda.org/conda-forge/noarch/xyzservices-2024.9.0-pyhd8ed1ab_0.conda#156c91e778c1d4d57b709f8c5333fd06 -https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h3b0a872_6.conda#113506c8d2d558e733f5c38f6bf08c50 +https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.5-h3b0a872_7.conda#3947a35e916fcc6b9825449affbf4214 https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30c2c15b82aacb07f9c09e28ff2275 https://conda.anaconda.org/conda-forge/noarch/zipp-3.21.0-pyhd8ed1ab_0.conda#fee389bf8a4843bd7a2248ce11b7f188 https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_0.conda#1bb1ef9806a9a20872434f58b3e7fc1a https://conda.anaconda.org/conda-forge/noarch/asttokens-2.4.1-pyhd8ed1ab_0.conda#5f25798dcefd8252ce5f9dc494d5f571 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.7.0-h858c4ad_7.conda#1698a4867ecd97931d1bb743428686ec +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.7.1-h3a84f74_3.conda#e7a54821aaa774cfd64efcd45114a4d7 https://conda.anaconda.org/conda-forge/linux-64/azure-core-cpp-1.14.0-h5cfcd09_0.conda#0a8838771cc2e985cd295e01ae83baf1 https://conda.anaconda.org/conda-forge/noarch/babel-2.16.0-pyhd8ed1ab_0.conda#6d4e9ecca8d88977147e109fc7053184 https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.3-pyha770c72_0.conda#332493000404d8411859539a5a630865 https://conda.anaconda.org/conda-forge/noarch/bleach-6.2.0-pyhd8ed1ab_0.conda#461bcfab8e65c166e297222ae919a2d4 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py312h06ac9bb_0.conda#a861504bbea4161a9170b85d4d2be840 +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py313hfab6e84_0.conda#ce6386a5892ef686d6d680c345c40ad1 https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.4.1-ha728647_2.conda#dab65ce7f9da0b25f53f0ec0d37ee09c https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 -https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.4-py312h178313f_0.conda#a32fbd2322865ac80c7db74c553f5306 -https://conda.anaconda.org/conda-forge/linux-64/cytoolz-1.0.0-py312h66e93f0_1.conda#a921e2fe122e7f38417b9b17c7a13343 +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.7-py313h8060acc_0.conda#e87423953e8fc4eaab4a80e3e82c256e +https://conda.anaconda.org/conda-forge/linux-64/cytoolz-1.0.0-py313h536fd9c_1.conda#f536889754b62dad2e509cb858f525ee https://conda.anaconda.org/conda-forge/noarch/fire-0.7.0-pyhd8ed1ab_0.conda#c8eefdf1e822c56a6034602e67bc92a5 -https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.54.1-py312h178313f_1.conda#bbbf5fa5cab622c33907bc8d7eeea9f7 +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.55.0-py313h8060acc_0.conda#0ff3a44b54d02157f6e99074432b7396 https://conda.anaconda.org/conda-forge/linux-64/freeglut-3.2.2-ha6d2627_3.conda#84ec3f5b46f3076be49f2cf3f1cfbf02 https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.1-pyhd8ed1ab_1.conda#358c17429c97883b2cb9ab5f64bc161b https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_0.tar.bz2#b748fbf7060927a6e82df7cb5ee8f097 @@ -261,28 +274,28 @@ https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.3-nompi_hdf9ad27_105.c https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-8.5.0-pyha770c72_0.conda#54198435fce4d64d8a89af22573012a8 https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.4.5-pyhd8ed1ab_0.conda#c808991d29b9838fb4d96ce8267ec9ec https://conda.anaconda.org/conda-forge/noarch/isort-5.13.2-pyhd8ed1ab_0.conda#1d25ed2b95b92b026aaa795eabec8d91 -https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.1-pyhd8ed1ab_0.conda#81a3be0b2023e1ea8555781f0ad904a2 +https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.2-pyhff2d567_0.conda#11ead81b00e0f7cc901fceb7ccfb92c1 https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.4-pyhd8ed1ab_0.conda#7b86ecb7d3557821c649b3c31e3eb9f2 https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.7.2-pyh31011fe_1.conda#0a2980dada0dd7fd0998f0342308b1b1 https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.3.0-pyhd8ed1ab_1.conda#afcd1b53bcac8844540358e33f33d28f https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 +https://conda.anaconda.org/conda-forge/linux-64/libglu-9.0.3-h03adeef_0.conda#b1df5affe904efe82ef890826b68881d https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.31.0-h804f50b_0.conda#35ab838423b60f233391eb86d324a830 -https://conda.anaconda.org/conda-forge/linux-64/lxml-5.3.0-py312he28fd5a_2.conda#3acf38086326f49afed094df4ba7c9d9 +https://conda.anaconda.org/conda-forge/linux-64/lxml-5.3.0-py313h6eb7059_2.conda#48d1a2d9b1f12ff5180ffb4154050c48 https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_0.conda#779345c95648be40d22aaa89de7d4254 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.9.1-pyhd8ed1ab_0.conda#dfe0528d0f1c16c1f7c528ea5536ab30 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py312heda63a1_0.conda#d8285bea2a350f63fab23bf460221f3f +https://conda.anaconda.org/conda-forge/linux-64/numpy-2.1.3-py313h4bf6692_0.conda#17bcf851cceab793dad11ab8089d4bc4 https://conda.anaconda.org/conda-forge/linux-64/openldap-2.6.8-hedd0468_0.conda#dcd0ed5147d8876b0848a552b416ce76 https://conda.anaconda.org/conda-forge/noarch/partd-1.4.2-pyhd8ed1ab_0.conda#0badf9c54e24cecfb0ad2f99d680c163 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_0.conda#629f3203c99b32e0988910c93e77f3b6 -https://conda.anaconda.org/conda-forge/linux-64/pillow-11.0.0-py312h7b63e92_0.conda#385f46a4df6f97892503a841121a9acf -https://conda.anaconda.org/conda-forge/noarch/pip-24.3.1-pyh8b19718_0.conda#5dd546fe99b44fda83963d15f84263b7 +https://conda.anaconda.org/conda-forge/linux-64/pillow-11.0.0-py313h2d7ed13_0.conda#0d95e1cda6bf9ce501e751c02561204e https://conda.anaconda.org/conda-forge/linux-64/poppler-24.08.0-h47131b8_1.conda#0854b9ff0cc10a1f6f67b0f352b8e75a https://conda.anaconda.org/conda-forge/linux-64/proj-9.5.0-h12925eb_0.conda#8c29983ebe50cc7e0998c34bc7614222 https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.48-pyha770c72_0.conda#4c05134c48b6a74f33bbb9938e4a115e https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.3-pyhd8ed1ab_0.conda#c03d61f31f38fdb9facf70c29958bf7a -https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0-pyhd8ed1ab_0.conda#2cf4264fffb9e6eff6031c5b6884d61c -https://conda.anaconda.org/conda-forge/linux-64/pyzmq-26.2.0-py312hbf22597_3.conda#746ce19f0829ec3e19c93007b1a224d3 +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhff2d567_0.conda#b6dfd90a2141e573e4b6a81630b56df5 +https://conda.anaconda.org/conda-forge/linux-64/pyzmq-26.2.0-py313h8e95178_3.conda#8ab50c9c9c3824ac0ffac9e9dcf5619e https://conda.anaconda.org/conda-forge/noarch/rdflib-7.1.1-pyh0610db2_0.conda#325219de79481bcf5b6446d327e3d492 https://conda.anaconda.org/conda-forge/noarch/referencing-0.35.1-pyhd8ed1ab_0.conda#0fc8b52192a8898627c3efae1003e9f6 https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.4.0-pyhd8ed1ab_0.conda#f1acf5fdefa8300de697982bcb1761c9 @@ -291,110 +304,109 @@ https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.t https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.27.1-pyhd8ed1ab_0.conda#dae21509d62aa7bf676279ced3edcb3f https://conda.anaconda.org/conda-forge/noarch/webob-1.8.9-pyhd8ed1ab_0.conda#ff98f23ad74d2a3256debcd9df65d37d https://conda.anaconda.org/conda-forge/noarch/yamale-5.2.1-pyhca7485f_0.conda#c089f90a086b6214c5606368d0d3bad0 -https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.29.3-hbc793f2_2.conda#3ac9933695a731e6507eef6c3704c10f +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.29.4-h21d7256_1.conda#963a310ba64fd6a113eb4f7fcf89f935 https://conda.anaconda.org/conda-forge/linux-64/azure-identity-cpp-1.10.0-h113e628_0.conda#73f73f60854f325a55f1d31459f2ab73 https://conda.anaconda.org/conda-forge/linux-64/azure-storage-common-cpp-12.8.0-h736e048_1.conda#13de36be8de3ae3f05ba127631599213 https://conda.anaconda.org/conda-forge/noarch/cattrs-24.1.2-pyhd8ed1ab_0.conda#ac582de2324988b79870b50c89c91c75 -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.4-py312hc0a28a1_1.conda#990033147b0a998e756eaaed6b28f48d -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.0-py312h68727a3_2.conda#ff28f374b31937c048107521c814791e -https://conda.anaconda.org/conda-forge/linux-64/cryptography-43.0.3-py312hda17c39_0.conda#2abada8c216dd6e32514535a3fa245d4 +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.4-py313ha014f3b_1.conda#b20667f9b1d016c1141051a433f76dfc +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.3.1-py313h33d0bda_0.conda#6b6768e7c585d7029f79a04cbc4cbff0 +https://conda.anaconda.org/conda-forge/linux-64/cryptography-43.0.3-py313h6556f6e_0.conda#4df31328181600b08e18f709269d6f52 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2024.11.2-pyhff2d567_1.conda#ae2be36dab764e655a22f240837cef75 https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.3-h77b800c_3.conda#4eb52aecb43e7c72f8e4fca0c386354e -https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-8.5.0-hd8ed1ab_0.conda#2a92e152208121afadf85a5e1f3a5f4d https://conda.anaconda.org/conda-forge/linux-64/jasper-4.2.4-h536e39c_0.conda#9518ab7016cf4564778aef08b6bd8792 https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2024.10.1-pyhd8ed1ab_0.conda#720745920222587ef942acfbc578b584 https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_0.conda#a14218cfb29662b4a19ceb04e93e298e https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.3-hf8d3e68_2.conda#ffe68c611ae0ccfda4e7a605195e22b3 https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-storage-2.31.0-h0121fbd_0.conda#568d6a09a6ed76337a7b97c84ae7c0f8 https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h135f659_114.conda#a908e463c710bd6b10a9eaa89fdf003c -https://conda.anaconda.org/conda-forge/linux-64/libpq-17.0-h04577a9_4.conda#392cae2a58fbcb9db8c2147c6d6d1620 +https://conda.anaconda.org/conda-forge/linux-64/libpq-17.1-h04577a9_0.conda#c2560bae9f56de89b8c50355f7c84910 https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h1b4f908_11.conda#43a7f3df7d100e8fc280e6636680a870 -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py312hf9745cd_1.conda#8bce4f6caaf8c5448c7ac86d87e26b4b +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.2.3-py313ha87cce1_1.conda#c5d63dd501db554b84a30dea33824164 https://conda.anaconda.org/conda-forge/linux-64/pango-1.54.0-h4c5309f_1.conda#7df02e445367703cd87a574046e3a6f0 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba https://conda.anaconda.org/conda-forge/noarch/pylint-3.3.1-pyhd8ed1ab_0.conda#2a3426f75e2172c932131f4e3d51bcf4 -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.7.0-py312he630544_0.conda#427799f15b36751761941f4cbd7d780f +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.7.0-py313hdb96ca5_0.conda#2a0d20f16832a170218b474bcec57acf https://conda.anaconda.org/conda-forge/noarch/pytest-cov-6.0.0-pyhd8ed1ab_0.conda#cb8a11b6d209e3d85e5094bdbd9ebd9c https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.1.5-pyhd8ed1ab_0.conda#ecd5e850bcd3eca02143e7df030ee50f https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.1.1-pyhd8ed1ab_0.conda#52b91ecba854d55b28ad916a8b10da24 https://conda.anaconda.org/conda-forge/noarch/pytest-mock-3.14.0-pyhd8ed1ab_0.conda#4b9b5e086812283c052a9105ab1e254e https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.6.1-pyhd8ed1ab_0.conda#b39568655c127a9c4a44d178ac99b6d0 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.14.1-py312h62794b6_1.conda#b43233a9e2f62fb94affe5607ea79473 -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.6-py312h391bc85_2.conda#eb476b4975ea28ac12ff469063a71f5d +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.14.1-py313h27c5614_1.conda#c5c52b95724a6d4adb72499912eea085 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.6-py313h3f71f02_2.conda#dd0b742e8e61b8f15e4b64efcc103ad6 https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.2-pyhd8ed1ab_0.conda#e7df0fdd404616638df5ece6e69ba7af -https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py312h68727a3_5.conda#f9664ee31aed96c85b7319ab0a693341 -https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py312hef9b889_1.conda#8b7069e9792ee4e5b4919a7a306d2e67 -https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.407-h5cd358a_9.conda#1bba87c0e95867ad8ef2932d603ce7ee +https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py313h33d0bda_5.conda#5bcffe10a500755da4a71cc0fb62a420 +https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.23.0-py313h80202fe_1.conda#c178558ff516cd507763ffee230c20b2 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.449-h1a02111_2.conda#109ff9aa7347ca004a3f496a5160cdb9 https://conda.anaconda.org/conda-forge/linux-64/azure-storage-blobs-cpp-12.13.0-h3cf044e_1.conda#7eb66060455c7a47d9dcdbfa9f46579b https://conda.anaconda.org/conda-forge/noarch/bokeh-3.6.1-pyhd8ed1ab_0.conda#e88d74bb7b9b89d4c9764286ceb94cc9 -https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.3.0-py312hc0a28a1_0.conda#8b5b812d4c18cb37bda7a7c8d3a6acb3 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2024.11.0-pyhd8ed1ab_0.conda#75c96f0655908f596a57be60251b78d4 +https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.3.0-py313ha014f3b_0.conda#aecffd7a21d698e374487644ce67d6eb https://conda.anaconda.org/conda-forge/linux-64/eccodes-2.38.3-h8bb6dbc_1.conda#73265d4acc551063cc5c5beab37f33c5 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h6470451_5.conda#1483ba046164be27df7f6eddbcec3a12 https://conda.anaconda.org/conda-forge/noarch/identify-2.6.2-pyhd8ed1ab_0.conda#636950f839e065401e2031624a414f0b https://conda.anaconda.org/conda-forge/noarch/ipython-8.29.0-pyh707e725_0.conda#56db21d7d51410fcfbfeca3d1a6b4269 https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.23.0-pyhd8ed1ab_0.conda#da304c192ad59975202859b367d0f6a2 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-core-3.9.3-hd5b9bfb_2.conda#b70c6b3de9d4779d40dc3194f3958889 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-core-3.10.0-hef9eae6_1.conda#6271d1929f8c1964f5f1d56a7f996b19 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.58.4-hc0ffecb_0.conda#83f045969988f5c7a65f3950b95a8b35 -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.9.2-py312hd3ec401_2.conda#2380c9ba933ffaac9ad16d8eac8e3318 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.9.2-py313h129903b_2.conda#71d8f34a558d7e4d6656679c609b65d5 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_h22f9119_106.conda#5b911bfe75855326bae6857451268e59 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.7.1-nompi_py312h21d6d8e_102.conda#9049ba34261ce7106220711d313fcf61 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-17.0-h1122569_4.conda#028ea131f116f13bb2a4a382b5863a04 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.7.1-nompi_py313h2a70696_102.conda#f4e34c42e744348631b5c6c37efe7cd4 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-17.1-h1122569_0.conda#10dcb54ee745ee2a51d5370ba8e5657e https://conda.anaconda.org/conda-forge/noarch/pyopenssl-24.2.1-pyhd8ed1ab_2.conda#85fa2fdd26d5a38792eb57bc72463f07 https://conda.anaconda.org/conda-forge/noarch/pytest-html-4.1.1-pyhd8ed1ab_0.conda#4d2040212307d18392a2687772b3a96d +https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py313ha014f3b_3.conda#041b8326743c64bd02b8c0f34f05e1ef https://conda.anaconda.org/conda-forge/noarch/urllib3-2.2.3-pyhd8ed1ab_0.conda#6b55867f385dd762ed99ea687af32a69 https://conda.anaconda.org/conda-forge/noarch/xarray-2024.10.0-pyhd8ed1ab_0.conda#53e365732dfa053c4d19fc6b927392c4 https://conda.anaconda.org/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.12.0-ha633028_1.conda#7c1980f89dd41b097549782121a73490 -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.24.0-py312hf9745cd_0.conda#ea213e31805199cb7d0da457b879ceed +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.24.0-py313ha87cce1_0.conda#44c2091019480603a885aa01e7b710e7 https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.10.0-pyhd8ed1ab_0.conda#9437cfe346eab83b011b4def99f0e879 -https://conda.anaconda.org/conda-forge/noarch/distributed-2024.11.0-pyhd8ed1ab_0.conda#497f3535cbb69cd2f02158e2e18ee0bb +https://conda.anaconda.org/conda-forge/noarch/distributed-2024.11.2-pyhff2d567_1.conda#171408408370e59126dc3e39352c6218 https://conda.anaconda.org/conda-forge/linux-64/esmf-8.6.1-nompi_h4441c20_3.conda#1afc1e85414e228916732df2b8c5d93b -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.9.3-py312h1299960_2.conda#5870fd19530dc5a58f101001f5f91c5e +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.10.0-py313h7cbee32_1.conda#f6c287930ef6b23a23cb1952e19d2aa9 https://conda.anaconda.org/conda-forge/linux-64/graphviz-12.0.0-hba01fac_0.conda#953e31ea00d46beb7e64a79fc291ec44 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-fits-3.9.3-h2db6552_2.conda#80f15b889bb72a1077a58c057316969c -https://conda.anaconda.org/conda-forge/linux-64/libgdal-grib-3.9.3-hc3b29a1_2.conda#554eea65ee24d8abea3a4b9aed343999 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-hdf4-3.9.3-hd5ecb85_2.conda#04ae2626a06e48fcd571be05570da73a -https://conda.anaconda.org/conda-forge/linux-64/libgdal-hdf5-3.9.3-h6283f77_2.conda#b061e75ec58ab3a6e7d43d15429972a2 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-jp2openjpeg-3.9.3-h1b2c38e_2.conda#647ab247f197516aaedfa3777380b7cb -https://conda.anaconda.org/conda-forge/linux-64/libgdal-pdf-3.9.3-h600f43f_2.conda#6398f3e1cc2f25c2c3e0b372b09a76e7 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-pg-3.9.3-h5e77dd0_2.conda#bb2d1fc62cbad382d9bf5957383d065b -https://conda.anaconda.org/conda-forge/linux-64/libgdal-postgisraster-3.9.3-h5e77dd0_2.conda#919ec2fbbfc11fe80bf7f4bcc586c737 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-xls-3.9.3-h03c987c_2.conda#b638ddb82a5f0f289e7e5968fae6e4fe +https://conda.anaconda.org/conda-forge/linux-64/libgdal-fits-3.10.0-he1674de_1.conda#415c6f3d27f39731b38f89db57f785f7 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-grib-3.10.0-ha360943_1.conda#c8ec329a2a0e09deae512b24bebba974 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-hdf4-3.10.0-h380f24e_1.conda#06b598afa8b4d73818d6ae5fbf57cce1 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-hdf5-3.10.0-hefe6d7a_1.conda#ff882b327028dd49f9db1eb0c4ca4225 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-jp2openjpeg-3.10.0-h9fdfae1_1.conda#b5284debccc01a949a6d744e4e793b2d +https://conda.anaconda.org/conda-forge/linux-64/libgdal-pdf-3.10.0-h697c966_1.conda#0b8c19eaf166d18bb1f10d759038825f +https://conda.anaconda.org/conda-forge/linux-64/libgdal-pg-3.10.0-h5cc4e75_1.conda#4749862355fee05f7a5a7c41a2cfac7d +https://conda.anaconda.org/conda-forge/linux-64/libgdal-postgisraster-3.10.0-h5cc4e75_1.conda#8d3f0806eb386761975cff7345858c6c +https://conda.anaconda.org/conda-forge/linux-64/libgdal-xls-3.10.0-h1e14832_1.conda#5419c1134aabe809f67059808be80195 https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.1-pyhd8ed1ab_0.conda#bcdbeb2b693eba886583a907840c6421 https://conda.anaconda.org/conda-forge/noarch/nbformat-5.10.4-pyhd8ed1ab_0.conda#0b57b5368ab7fc7cdc9e3511fa867214 https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/noarch/pre-commit-4.0.1-pyha770c72_0.conda#5971cc64048943605f352f7f8612de6c -https://conda.anaconda.org/conda-forge/linux-64/python-eccodes-2.37.0-py312hc0a28a1_0.conda#476b0357e207e10d2b7b13ed82156e6d -https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py312hc0a28a1_3.conda#81bbcb20ea4a53b05a8cf51f31496038 +https://conda.anaconda.org/conda-forge/linux-64/python-eccodes-2.37.0-py313ha014f3b_0.conda#b28717a6d595cdc42737d6669d422b1d https://conda.anaconda.org/conda-forge/noarch/requests-2.32.3-pyhd8ed1ab_0.conda#5ede4753180c7a550a443c430dc8ab52 -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.26.2-h8a7cb20_8.conda#610687d178a4cd4942dc6993064ae61d +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.26.2-h84bbdfb_10.conda#c9ad5ee546eba614b7fe7b420f6b7763 https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.9.0-pyhd8ed1ab_0.conda#a201de7d36907f2355426e019168d337 https://conda.anaconda.org/conda-forge/noarch/esmpy-8.6.1-pyhc1e730c_0.conda#25a9661177fd68bfdb4314fd658e5c3b -https://conda.anaconda.org/conda-forge/noarch/iris-3.10.0-pyha770c72_2.conda#5d8984ceb5fdf85110ca7108114ecc18 -https://conda.anaconda.org/conda-forge/linux-64/libarrow-18.0.0-h3e543c6_4_cpu.conda#98ca983152358b918afebf2abe9e5ca9 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-kea-3.9.3-h1df15e4_2.conda#f11131205273b99ec9a8742e1e9ae4d9 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-netcdf-3.9.3-hf2d2f32_2.conda#3b34da7734c870baec2853b278a7f581 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-tiledb-3.9.3-h4a3bace_2.conda#9b59f8fc89321fe0c0cce1bb48fac282 +https://conda.anaconda.org/conda-forge/noarch/iris-3.11.0-pyha770c72_0.conda#a5e36260789ce92074c3736533ecdd61 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-18.0.0-h3b997a5_7_cpu.conda#32897a50e7f68187c4a524c439c0943c +https://conda.anaconda.org/conda-forge/linux-64/libgdal-kea-3.10.0-h38e673a_1.conda#1b4358b735ef045fbd87d2ec3341a6a2 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-netcdf-3.10.0-hba670d9_1.conda#85699d0969e7c92ba75c7bb0e7cbed19 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-tiledb-3.10.0-hec57c18_1.conda#eb39051813bc34137bff2e4ad8dfe64e https://conda.anaconda.org/conda-forge/noarch/nbclient-0.10.0-pyhd8ed1ab_0.conda#15b51397e0fe8ea7d7da60d83eb76ebc https://conda.anaconda.org/conda-forge/noarch/pooch-1.8.2-pyhd8ed1ab_0.conda#8dab97d8a9616e07d779782995710aed -https://conda.anaconda.org/conda-forge/linux-64/pydot-3.0.2-py312h7900ff3_0.conda#a972ba77217a2cac592c41dd3cc56dfd +https://conda.anaconda.org/conda-forge/linux-64/pydot-3.0.2-py313h78bf25f_0.conda#45f3a293c1709b761bd450917cecd8c6 https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.2.1-pyhd8ed1ab_0.conda#c6089540fed51a9a829aa19590fa925b https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyhd8ed1ab_4.conda#f481c17430f801e68ee3b57cc30ecd2e https://conda.anaconda.org/conda-forge/noarch/iris-grib-0.20.0-pyhd8ed1ab_1.conda#d8dced41fc56982c81190ba0eb10c3de -https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-18.0.0-h5888daf_4_cpu.conda#86298c079658bed57c5590690a0fb418 -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.9.3-ha770c72_2.conda#23ab37bf861be73033b5b0ad4c7d80b1 -https://conda.anaconda.org/conda-forge/linux-64/libparquet-18.0.0-h6bd9018_4_cpu.conda#19973fe63c087ef5e119a2826011efb4 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-acero-18.0.0-h5888daf_7_cpu.conda#786a275d019708cd1c963b12a8fb0c72 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.10.0-ha770c72_1.conda#f32b9e97d0394dcc2f6f5758dc18afa1 +https://conda.anaconda.org/conda-forge/linux-64/libparquet-18.0.0-h6bd9018_7_cpu.conda#687870f7d9cba5262fdd7e730e9e9ba8 https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.16.4-pyhd8ed1ab_1.conda#e2d2abb421c13456a9a9f80272fdf543 https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 -https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.8.0-pyhd8ed1ab_0.conda#fba377622e74ee0bbeb8ccae9fa593d3 -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-18.0.0-py312h01725c0_1_cpu.conda#c8ae967c39337603035d59c8994c23f9 -https://conda.anaconda.org/conda-forge/linux-64/fiona-1.10.1-py312h5aa26c2_1.conda#4a30f4277a1894928a7057d0e14c1c95 -https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-18.0.0-h5888daf_4_cpu.conda#6ac53d3f10c9d88ade8f9fe0f515a0db +https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.9.0-pyhd8ed1ab_0.conda#177a9651dc31c11a81eddc2a5e2e524e +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-core-18.0.0-py313he5f92c8_1_cpu.conda#34918674d521ab777f11ab3c1f2ab797 +https://conda.anaconda.org/conda-forge/linux-64/fiona-1.10.1-py313hab20ce0_2.conda#c0cf01c18f0c4f38c84cd906409ec5e4 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-dataset-18.0.0-h5888daf_7_cpu.conda#a742b9a0452b55020ccf662721c1ce44 https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.16.4-hd8ed1ab_1.conda#37cec2cf68f4c09563d8bc833791096b -https://conda.anaconda.org/conda-forge/linux-64/libarrow-substrait-18.0.0-h5c8f2c3_4_cpu.conda#24f60812bdd87979ea1c6477f2f38d3b +https://conda.anaconda.org/conda-forge/linux-64/libarrow-substrait-18.0.0-h5c8f2c3_7_cpu.conda#be76013fa3fdaec2c0c504e6fdfd282d https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.16.4-hd8ed1ab_1.conda#ab83e3b9ca2b111d8f332e9dc8b2170f -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-18.0.0-py312h7900ff3_1.conda#ea33ac754057779cd2df785661486310 -https://conda.anaconda.org/conda-forge/noarch/dask-expr-1.1.17-pyhd8ed1ab_0.conda#4f75a3a76e9f693fc33be59485f46fcf -https://conda.anaconda.org/conda-forge/noarch/dask-2024.11.0-pyhd8ed1ab_0.conda#9a25bf7e2a910e85209218896f2adeb9 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-18.0.0-py313h78bf25f_1.conda#7ce246ff42b7797a9c270964c94faf05 +https://conda.anaconda.org/conda-forge/noarch/dask-expr-1.1.19-pyhd8ed1ab_0.conda#09ea33eb6525cc703ce1d39c88378320 +https://conda.anaconda.org/conda-forge/noarch/dask-2024.11.2-pyhff2d567_1.conda#4ea56955c9922ac99c35d0784cffeb96 https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.11.0-pyhd8ed1ab_1.conda#86286b197e33e3b034416c18ba0f574c https://conda.anaconda.org/conda-forge/noarch/autodocsumm-0.2.14-pyhd8ed1ab_0.conda#351a11ac1215eb4f6c5b82e30070277a https://conda.anaconda.org/conda-forge/noarch/nbsphinx-0.9.5-pyhd8ed1ab_0.conda#b808b8a0494c5cca76200c73e260a060 @@ -405,5 +417,5 @@ https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_0.conda#d6e5ea5fe00164ac6c2dcc5d76a42192 https://conda.anaconda.org/conda-forge/noarch/sphinx-8.1.3-pyhd8ed1ab_0.conda#05706dd5a145a9c91861495cd435409a https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.10-pyhd8ed1ab_0.conda#e507335cb4ca9cff4c3d0fa9cdab255e -# pip scitools-iris @ https://files.pythonhosted.org/packages/13/f9/492f73d8cb5cc6a4552448e2583690e918d8ed0c7dad661fb118340ab127/scitools_iris-3.10.0-py3-none-any.whl#sha256=01f99d9cabde69536f21ca31213e5497e1c7d62cd7222e06bfa05885318c9169 +# pip scitools-iris @ https://files.pythonhosted.org/packages/20/89/109d116f778fd148782598eb1796db00d47de8ca0d68503d248b55154581/scitools_iris-3.11.0-py3-none-any.whl#sha256=97bb7d7e349808684a5326a1ec06a459702a2b4f435c9a1502378d41e24a32f3 # pip esmvaltool-sample-data @ https://files.pythonhosted.org/packages/58/fa/4ecc84665e0ed04c8c4c797405c19c12900bdba6438ab2f5541bf8aa1d42/ESMValTool_sample_data-0.0.3-py3-none-any.whl#sha256=81f0f02182eacb3b639cb207abae5ac469c6dd83fb6dfe6d2430c69723d85461 From d0bfb58cecf4daa9e1587342919decfc0623925f Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 25 Nov 2024 14:48:27 +0100 Subject: [PATCH 03/17] Miscellaneous lazy preprocessor improvements (#2520) Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_area.py | 66 +++++++----- esmvalcore/preprocessor/_io.py | 2 + esmvalcore/preprocessor/_mask.py | 32 +----- esmvalcore/preprocessor/_regrid.py | 31 +----- esmvalcore/preprocessor/_shared.py | 78 ++++++++++++++ esmvalcore/preprocessor/_time.py | 4 + tests/unit/preprocessor/_area/test_area.py | 37 +++++-- tests/unit/preprocessor/_mask/test_mask.py | 25 ----- .../_regrid/test_extract_levels.py | 56 ---------- tests/unit/preprocessor/_time/test_time.py | 14 ++- tests/unit/preprocessor/test_shared.py | 102 ++++++++++++++++++ 11 files changed, 270 insertions(+), 177 deletions(-) diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index a47ca29892..7d8d867155 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -21,6 +21,8 @@ from iris.exceptions import CoordinateNotFoundError from esmvalcore.preprocessor._shared import ( + apply_mask, + get_dims_along_axes, get_iris_aggregator, get_normalized_cube, preserve_float_dtype, @@ -188,8 +190,8 @@ def _extract_irregular_region( cube = cube[..., i_slice, j_slice] selection = selection[i_slice, j_slice] # Mask remaining coordinates outside region - mask = da.broadcast_to(~selection, cube.shape) - cube.data = da.ma.masked_where(mask, cube.core_data()) + horizontal_dims = get_dims_along_axes(cube, ["X", "Y"]) + cube.data = apply_mask(~selection, cube.core_data(), horizontal_dims) return cube @@ -857,31 +859,45 @@ def _mask_cube(cube: Cube, masks: dict[str, np.ndarray]) -> Cube: _cube.add_aux_coord( AuxCoord(id_, units="no_unit", long_name="shape_id") ) - mask = da.broadcast_to(mask, _cube.shape) - _cube.data = da.ma.masked_where(~mask, _cube.core_data()) + horizontal_dims = get_dims_along_axes(cube, axes=["X", "Y"]) + _cube.data = apply_mask(~mask, _cube.core_data(), horizontal_dims) cubelist.append(_cube) result = fix_coordinate_ordering(cubelist.merge_cube()) - if cube.cell_measures(): - for measure in cube.cell_measures(): - # Cell measures that are time-dependent, with 4 dimension and - # an original shape of (time, depth, lat, lon), need to be - # broadcasted to the cube with 5 dimensions and shape - # (time, shape_id, depth, lat, lon) - if measure.ndim > 3 and result.ndim > 4: - data = measure.core_data() - data = da.expand_dims(data, axis=(1,)) - data = da.broadcast_to(data, result.shape) - measure = iris.coords.CellMeasure( + for measure in cube.cell_measures(): + # Cell measures that are time-dependent, with 4 dimension and + # an original shape of (time, depth, lat, lon), need to be + # broadcast to the cube with 5 dimensions and shape + # (time, shape_id, depth, lat, lon) + if measure.ndim > 3 and result.ndim > 4: + data = measure.core_data() + if result.has_lazy_data(): + # Make the cell measure lazy if the result is lazy. + cube_chunks = cube.lazy_data().chunks + chunk_dims = cube.cell_measure_dims(measure) + data = da.asarray( data, - standard_name=measure.standard_name, - long_name=measure.long_name, - units=measure.units, - measure=measure.measure, - var_name=measure.var_name, - attributes=measure.attributes, + chunks=tuple(cube_chunks[i] for i in chunk_dims), ) - add_cell_measure(result, measure, measure.measure) - if cube.ancillary_variables(): - for ancillary_variable in cube.ancillary_variables(): - add_ancillary_variable(result, ancillary_variable) + chunks = result.lazy_data().chunks + else: + chunks = None + dim_map = get_dims_along_axes(result, ["T", "Z", "Y", "X"]) + data = iris.util.broadcast_to_shape( + data, + result.shape, + dim_map=dim_map, + chunks=chunks, + ) + measure = iris.coords.CellMeasure( + data, + standard_name=measure.standard_name, + long_name=measure.long_name, + units=measure.units, + measure=measure.measure, + var_name=measure.var_name, + attributes=measure.attributes, + ) + add_cell_measure(result, measure, measure.measure) + for ancillary_variable in cube.ancillary_variables(): + add_ancillary_variable(result, ancillary_variable) return result diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 5f83b1946c..83f4d9bae5 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -22,6 +22,7 @@ from esmvalcore.cmor.check import CheckLevels from esmvalcore.esgf.facets import FACETS from esmvalcore.iris_helpers import merge_cube_attributes +from esmvalcore.preprocessor._shared import _rechunk_aux_factory_dependencies from .._task import write_ncl_settings @@ -392,6 +393,7 @@ def concatenate(cubes, check_level=CheckLevels.DEFAULT): cubes = _sort_cubes_by_time(cubes) _fix_calendars(cubes) cubes = _check_time_overlaps(cubes) + cubes = [_rechunk_aux_factory_dependencies(cube) for cube in cubes] result = _concatenate_cubes(cubes, check_level=check_level) if len(result) == 1: diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py index 1896475704..1f1d0ddc00 100644 --- a/esmvalcore/preprocessor/_mask.py +++ b/esmvalcore/preprocessor/_mask.py @@ -9,8 +9,7 @@ import logging import os -from collections.abc import Iterable -from typing import Literal, Optional +from typing import Literal import cartopy.io.shapereader as shpreader import dask.array as da @@ -22,7 +21,7 @@ from iris.cube import Cube from iris.util import rolling_window -from esmvalcore.preprocessor._shared import get_array_module +from esmvalcore.preprocessor._shared import apply_mask from ._supplementary_vars import register_supplementaries @@ -61,24 +60,6 @@ def _get_fx_mask( return inmask -def _apply_mask( - mask: np.ndarray | da.Array, - array: np.ndarray | da.Array, - dim_map: Optional[Iterable[int]] = None, -) -> np.ndarray | da.Array: - """Apply a (broadcasted) mask on an array.""" - npx = get_array_module(mask, array) - if dim_map is not None: - if isinstance(array, da.Array): - chunks = array.chunks - else: - chunks = None - mask = iris.util.broadcast_to_shape( - mask, array.shape, dim_map, chunks=chunks - ) - return npx.ma.masked_where(mask, array) - - @register_supplementaries( variables=["sftlf", "sftof"], required="prefer_at_least_one", @@ -145,7 +126,7 @@ def mask_landsea(cube: Cube, mask_out: Literal["land", "sea"]) -> Cube: landsea_mask = _get_fx_mask( ancillary_var.core_data(), mask_out, ancillary_var.var_name ) - cube.data = _apply_mask( + cube.data = apply_mask( landsea_mask, cube.core_data(), cube.ancillary_variable_dims(ancillary_var), @@ -212,7 +193,7 @@ def mask_landseaice(cube: Cube, mask_out: Literal["landsea", "ice"]) -> Cube: landseaice_mask = _get_fx_mask( ancillary_var.core_data(), mask_out, ancillary_var.var_name ) - cube.data = _apply_mask( + cube.data = apply_mask( landseaice_mask, cube.core_data(), cube.ancillary_variable_dims(ancillary_var), @@ -350,10 +331,7 @@ def _mask_with_shp(cube, shapefilename, region_indices=None): else: mask |= shp_vect.contains(region, x_p_180, y_p_90) - if cube.has_lazy_data(): - mask = da.array(mask) - - cube.data = _apply_mask( + cube.data = apply_mask( mask, cube.core_data(), cube.coord_dims("latitude") + cube.coord_dims("longitude"), diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 10511102e2..5bbed48dcf 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -32,6 +32,7 @@ from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid from esmvalcore.preprocessor._shared import ( + _rechunk_aux_factory_dependencies, get_array_module, get_dims_along_axes, preserve_float_dtype, @@ -1174,36 +1175,6 @@ def parse_vertical_scheme(scheme): return scheme, extrap_scheme -def _rechunk_aux_factory_dependencies( - cube: iris.cube.Cube, - coord_name: str, -) -> iris.cube.Cube: - """Rechunk coordinate aux factory dependencies. - - This ensures that the resulting coordinate has reasonably sized - chunks that are aligned with the cube data for optimal computational - performance. - """ - # Workaround for https://github.com/SciTools/iris/issues/5457 - try: - factory = cube.aux_factory(coord_name) - except iris.exceptions.CoordinateNotFoundError: - return cube - - cube = cube.copy() - cube_chunks = cube.lazy_data().chunks - for coord in factory.dependencies.values(): - coord_dims = cube.coord_dims(coord) - if coord_dims: - coord = coord.copy() - chunks = tuple(cube_chunks[i] for i in coord_dims) - coord.points = coord.lazy_points().rechunk(chunks) - if coord.has_bounds(): - coord.bounds = coord.lazy_bounds().rechunk(chunks + (None,)) - cube.replace_coord(coord) - return cube - - @preserve_float_dtype def extract_levels( cube: iris.cube.Cube, diff --git a/esmvalcore/preprocessor/_shared.py b/esmvalcore/preprocessor/_shared.py index 2355215800..adf45ca1c2 100644 --- a/esmvalcore/preprocessor/_shared.py +++ b/esmvalcore/preprocessor/_shared.py @@ -517,3 +517,81 @@ def get_dims_along_coords( """Get a tuple with the dimensions along one or more coordinates.""" dims = {d for coord in coords for d in _get_dims_along(cube, coord)} return tuple(sorted(dims)) + + +def apply_mask( + mask: np.ndarray | da.Array, + array: np.ndarray | da.Array, + dim_map: Iterable[int], +) -> np.ma.MaskedArray | da.Array: + """Apply a (broadcasted) mask on an array. + + Parameters + ---------- + mask: + The mask to apply to array. + array: + The array to mask out. + dim_map : + A mapping of the dimensions of *mask* to their corresponding + dimension in *array*. + See :func:`iris.util.broadcast_to_shape` for additional details. + + Returns + ------- + np.ma.MaskedArray or da.Array: + A copy of the input array with the mask applied. + + """ + if isinstance(array, da.Array): + array_chunks = array.chunks + # If the mask is not a Dask array yet, we make it into a Dask array + # before broadcasting to avoid inserting a large array into the Dask + # graph. + mask_chunks = tuple(array_chunks[i] for i in dim_map) + mask = da.asarray(mask, chunks=mask_chunks) + else: + array_chunks = None + + mask = iris.util.broadcast_to_shape( + mask, array.shape, dim_map=dim_map, chunks=array_chunks + ) + + array_module = get_array_module(mask, array) + return array_module.ma.masked_where(mask, array) + + +def _rechunk_aux_factory_dependencies( + cube: iris.cube.Cube, + coord_name: str | None = None, +) -> iris.cube.Cube: + """Rechunk coordinate aux factory dependencies. + + This ensures that the resulting coordinate has reasonably sized + chunks that are aligned with the cube data for optimal computational + performance. + """ + # Workaround for https://github.com/SciTools/iris/issues/5457 + if coord_name is None: + factories = cube.aux_factories + else: + try: + factories = [cube.aux_factory(coord_name)] + except iris.exceptions.CoordinateNotFoundError: + return cube + + cube = cube.copy() + cube_chunks = cube.lazy_data().chunks + for factory in factories: + for coord in factory.dependencies.values(): + coord_dims = cube.coord_dims(coord) + if coord_dims: + coord = coord.copy() + chunks = tuple(cube_chunks[i] for i in coord_dims) + coord.points = coord.lazy_points().rechunk(chunks) + if coord.has_bounds(): + coord.bounds = coord.lazy_bounds().rechunk( + chunks + (None,) + ) + cube.replace_coord(coord) + return cube diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index b3e4ab5b0f..ac00f13d01 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -1286,6 +1286,10 @@ def timeseries_filter( # Apply filter (agg, agg_kwargs) = get_iris_aggregator(filter_stats, **operator_kwargs) agg_kwargs["weights"] = wgts + if cube.has_lazy_data(): + # Ensure the cube data chunktype is np.MaskedArray so rolling_window + # does not ignore a potential mask. + cube.data = da.ma.masked_array(cube.core_data()) cube = cube.rolling_window("time", agg, len(wgts), **agg_kwargs) return cube diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py index ec741b629a..4e5f19c28d 100644 --- a/tests/unit/preprocessor/_area/test_area.py +++ b/tests/unit/preprocessor/_area/test_area.py @@ -871,7 +871,10 @@ def test_extract_shape_natural_earth(make_testcube, ne_ocean_shapefile): np.testing.assert_array_equal(result.data.data, expected) -def test_extract_shape_fx(make_testcube, ne_ocean_shapefile): +@pytest.mark.parametrize("lazy", [True, False]) +def test_extract_shape_with_supplementaries( + make_testcube, ne_ocean_shapefile, lazy +): """Test for extracting a shape from NE file.""" expected = np.ones((5, 5)) cube = make_testcube @@ -888,6 +891,10 @@ def test_extract_shape_fx(make_testcube, ne_ocean_shapefile): var_name="sftgif", units="%", ) + if lazy: + cube.data = cube.lazy_data() + measure.data = measure.lazy_data() + ancillary_var.data = ancillary_var.lazy_data() cube.add_cell_measure(measure, (0, 1)) cube.add_ancillary_variable(ancillary_var, (0, 1)) result = extract_shape( @@ -895,17 +902,20 @@ def test_extract_shape_fx(make_testcube, ne_ocean_shapefile): ne_ocean_shapefile, crop=False, ) + assert result.has_lazy_data() is lazy np.testing.assert_array_equal(result.data.data, expected) assert result.cell_measures() - result_measure = result.cell_measure("cell_area").data - np.testing.assert_array_equal(measure.data, result_measure) + result_measure = result.cell_measure("cell_area") + assert result_measure.has_lazy_data() is lazy + np.testing.assert_array_equal(measure.data, result_measure.data) assert result.ancillary_variables() - result_ancillary_var = result.ancillary_variable( - "land_ice_area_fraction" - ).data - np.testing.assert_array_equal(ancillary_var.data, result_ancillary_var) + result_ancillary_var = result.ancillary_variable("land_ice_area_fraction") + assert result_ancillary_var.has_lazy_data() is lazy + np.testing.assert_array_equal( + ancillary_var.data, result_ancillary_var.data + ) def test_extract_shape_ne_check_nans(ne_ocean_shapefile): @@ -1471,7 +1481,8 @@ def test_meridional_statistics_invalid_norm_fail(make_testcube): meridional_statistics(make_testcube, "sum", normalize="x") -def test_time_dependent_volcello(): +@pytest.mark.parametrize("lazy", [True, False]) +def test_time_dependent_volcello(lazy): coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) data = np.ma.ones((2, 3, 2, 2)) @@ -1508,8 +1519,11 @@ def test_time_dependent_volcello(): volcello = iris.coords.CellMeasure( data, standard_name="ocean_volume", units="m3", measure="volume" ) + if lazy: + cube.data = cube.lazy_data() + volcello.data = volcello.lazy_data() cube.add_cell_measure(volcello, range(0, volcello.ndim)) - cube = extract_shape( + result = extract_shape( cube, "AR6", method="contains", @@ -1517,8 +1531,11 @@ def test_time_dependent_volcello(): decomposed=True, ids={"Acronym": ["EAO", "WAF"]}, ) + assert cube.has_lazy_data() is lazy + assert volcello.has_lazy_data() is lazy + assert result.has_lazy_data() is lazy - assert cube.shape == cube.cell_measure("ocean_volume").shape + assert result.shape == result.cell_measure("ocean_volume").shape if __name__ == "__main__": diff --git a/tests/unit/preprocessor/_mask/test_mask.py b/tests/unit/preprocessor/_mask/test_mask.py index 59b383c59a..dc6bfba162 100644 --- a/tests/unit/preprocessor/_mask/test_mask.py +++ b/tests/unit/preprocessor/_mask/test_mask.py @@ -9,7 +9,6 @@ import tests from esmvalcore.preprocessor._mask import ( - _apply_mask, _get_fx_mask, count_spells, mask_above_threshold, @@ -59,30 +58,6 @@ def setUp(self): ) self.fx_data = np.array([20.0, 60.0, 50.0]) - def test_apply_fx_mask_on_nonmasked_data(self): - """Test _apply_fx_mask func.""" - dummy_fx_mask = np.ma.array((True, False, True)) - app_mask = _apply_mask( - dummy_fx_mask, self.time_cube.data[0:3].astype("float64") - ) - fixed_mask = np.ma.array( - self.time_cube.data[0:3].astype("float64"), mask=dummy_fx_mask - ) - self.assert_array_equal(fixed_mask, app_mask) - - def test_apply_fx_mask_on_masked_data(self): - """Test _apply_fx_mask func.""" - dummy_fx_mask = np.ma.array((True, True, True)) - masked_data = np.ma.array( - self.time_cube.data[0:3].astype("float64"), - mask=np.ma.array((False, True, False)), - ) - app_mask = _apply_mask(dummy_fx_mask, masked_data) - fixed_mask = np.ma.array( - self.time_cube.data[0:3].astype("float64"), mask=dummy_fx_mask - ) - self.assert_array_equal(fixed_mask, app_mask) - def test_count_spells(self): """Test count_spells func.""" ref_spells = count_spells(self.time_cube.data, -1000.0, 0, 1) diff --git a/tests/unit/preprocessor/_regrid/test_extract_levels.py b/tests/unit/preprocessor/_regrid/test_extract_levels.py index e1b14b7a14..ec00d45438 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_levels.py +++ b/tests/unit/preprocessor/_regrid/test_extract_levels.py @@ -2,10 +2,8 @@ from unittest import mock -import dask.array as da import iris import numpy as np -from iris.aux_factory import HybridPressureFactory from numpy import ma import tests @@ -13,7 +11,6 @@ _MDI, VERTICAL_SCHEMES, _preserve_fx_vars, - _rechunk_aux_factory_dependencies, extract_levels, parse_vertical_scheme, ) @@ -349,56 +346,3 @@ def test_interpolation__masked(self): self.assert_array_equal(args[3], levels) # Check the _create_cube kwargs ... self.assertEqual(kwargs, dict()) - - -def test_rechunk_aux_factory_dependencies(): - delta = iris.coords.AuxCoord( - points=np.array([0.0, 1.0, 2.0], dtype=np.float64), - bounds=np.array( - [[-0.5, 0.5], [0.5, 1.5], [1.5, 2.5]], dtype=np.float64 - ), - long_name="level_pressure", - units="Pa", - ) - sigma = iris.coords.AuxCoord( - np.array([1.0, 0.9, 0.8], dtype=np.float64), - long_name="sigma", - units="1", - ) - surface_air_pressure = iris.coords.AuxCoord( - np.arange(4).astype(np.float64).reshape(2, 2), - long_name="surface_air_pressure", - units="Pa", - ) - factory = HybridPressureFactory( - delta=delta, - sigma=sigma, - surface_air_pressure=surface_air_pressure, - ) - - cube = iris.cube.Cube( - da.asarray( - np.arange(3 * 2 * 2).astype(np.float32).reshape(3, 2, 2), - chunks=(1, 2, 2), - ), - ) - cube.add_aux_coord(delta, 0) - cube.add_aux_coord(sigma, 0) - cube.add_aux_coord(surface_air_pressure, [1, 2]) - cube.add_aux_factory(factory) - - result = _rechunk_aux_factory_dependencies(cube, "air_pressure") - - # Check that the 'air_pressure' coordinate of the resulting cube has been - # rechunked: - assert ( - (1, 1, 1), - (2,), - (2,), - ) == result.coord("air_pressure").core_points().chunks - # Check that the original cube has not been modified: - assert ( - (3,), - (2,), - (2,), - ) == cube.coord("air_pressure").core_points().chunks diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index 6a9d78747d..e6c7ca09e6 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -1528,18 +1528,24 @@ def test_regrid_time_hour_no_divisor_of_24(cube_1d_time, freq): regrid_time(cube_1d_time, freq) -class TestTimeseriesFilter(tests.Test): +class TestTimeseriesFilter: """Tests for timeseries filter.""" + @pytest.fixture(autouse=True) def setUp(self): """Prepare tests.""" self.cube = _create_sample_cube() - def test_timeseries_filter_simple(self): + @pytest.mark.parametrize("lazy", [True, False]) + def test_timeseries_filter_simple(self, lazy): """Test timeseries_filter func.""" + if lazy: + self.cube.data = self.cube.lazy_data() filtered_cube = timeseries_filter( self.cube, 7, 14, filter_type="lowpass", filter_stats="sum" ) + if lazy: + assert filtered_cube.has_lazy_data() expected_data = np.array( [ 2.44824568, @@ -1569,14 +1575,14 @@ def test_timeseries_filter_timecoord(self): """Test missing time axis.""" new_cube = self.cube.copy() new_cube.remove_coord(new_cube.coord("time")) - with self.assertRaises(iris.exceptions.CoordinateNotFoundError): + with pytest.raises(iris.exceptions.CoordinateNotFoundError): timeseries_filter( new_cube, 7, 14, filter_type="lowpass", filter_stats="sum" ) def test_timeseries_filter_implemented(self): """Test a not implemented filter.""" - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): timeseries_filter( self.cube, 7, 14, filter_type="bypass", filter_stats="sum" ) diff --git a/tests/unit/preprocessor/test_shared.py b/tests/unit/preprocessor/test_shared.py index 90dd04135b..b449e1998f 100644 --- a/tests/unit/preprocessor/test_shared.py +++ b/tests/unit/preprocessor/test_shared.py @@ -8,6 +8,7 @@ import numpy as np import pytest from cf_units import Unit +from iris.aux_factory import HybridPressureFactory from iris.coords import AuxCoord from iris.cube import Cube @@ -15,12 +16,15 @@ from esmvalcore.preprocessor._shared import ( _compute_area_weights, _group_products, + _rechunk_aux_factory_dependencies, aggregator_accept_weights, + apply_mask, get_array_module, get_iris_aggregator, preserve_float_dtype, try_adding_calculated_cell_area, ) +from tests import assert_array_equal @pytest.mark.parametrize("operator", ["gmean", "GmEaN", "GMEAN"]) @@ -330,3 +334,101 @@ def test_try_adding_calculated_cell_area(): try_adding_calculated_cell_area(cube) assert cube.cell_measures("cell_area") + + +@pytest.mark.parametrize( + ["mask", "array", "dim_map", "expected"], + [ + ( + np.arange(2), + da.arange(2), + (0,), + da.ma.masked_array(np.arange(2), np.arange(2)), + ), + ( + da.arange(2), + np.arange(2), + (0,), + da.ma.masked_array(np.arange(2), np.arange(2)), + ), + ( + np.ma.masked_array(np.arange(2), mask=[1, 0]), + da.arange(2), + (0,), + da.ma.masked_array(np.ones(2), np.arange(2)), + ), + ( + np.ones((2, 5)), + da.zeros((2, 3, 5), chunks=(1, 2, 3)), + (0, 2), + da.ma.masked_array( + da.zeros((2, 3, 5), da.ones(2, 3, 5), chunks=(1, 2, 3)) + ), + ), + ( + np.arange(2), + np.ones((3, 2)), + (1,), + np.ma.masked_array(np.ones((3, 2)), mask=[[0, 1], [0, 1], [0, 1]]), + ), + ], +) +def test_apply_mask(mask, array, dim_map, expected): + result = apply_mask(mask, array, dim_map) + assert isinstance(result, type(expected)) + if isinstance(expected, da.Array): + assert result.chunks == expected.chunks + assert_array_equal(result, expected) + + +def test_rechunk_aux_factory_dependencies(): + delta = iris.coords.AuxCoord( + points=np.array([0.0, 1.0, 2.0], dtype=np.float64), + bounds=np.array( + [[-0.5, 0.5], [0.5, 1.5], [1.5, 2.5]], dtype=np.float64 + ), + long_name="level_pressure", + units="Pa", + ) + sigma = iris.coords.AuxCoord( + np.array([1.0, 0.9, 0.8], dtype=np.float64), + long_name="sigma", + units="1", + ) + surface_air_pressure = iris.coords.AuxCoord( + np.arange(4).astype(np.float64).reshape(2, 2), + long_name="surface_air_pressure", + units="Pa", + ) + factory = HybridPressureFactory( + delta=delta, + sigma=sigma, + surface_air_pressure=surface_air_pressure, + ) + + cube = iris.cube.Cube( + da.asarray( + np.arange(3 * 2 * 2).astype(np.float32).reshape(3, 2, 2), + chunks=(1, 2, 2), + ), + ) + cube.add_aux_coord(delta, 0) + cube.add_aux_coord(sigma, 0) + cube.add_aux_coord(surface_air_pressure, [1, 2]) + cube.add_aux_factory(factory) + + result = _rechunk_aux_factory_dependencies(cube, "air_pressure") + + # Check that the 'air_pressure' coordinate of the resulting cube has been + # rechunked: + assert ( + (1, 1, 1), + (2,), + (2,), + ) == result.coord("air_pressure").core_points().chunks + # Check that the original cube has not been modified: + assert ( + (3,), + (2,), + (2,), + ) == cube.coord("air_pressure").core_points().chunks From ce8714bfc2b3469d7d31133fd5886e259d1978ed Mon Sep 17 00:00:00 2001 From: Romain Beucher Date: Tue, 26 Nov 2024 05:54:13 +1000 Subject: [PATCH 04/17] Allows relative paths for diagnostic scripts. (#2329) Co-authored-by: Valeriu Predoi --- esmvalcore/_task.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 2a908583b6..66018c2789 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -3,6 +3,7 @@ import abc import contextlib import datetime +import importlib import logging import numbers import os @@ -386,9 +387,24 @@ def _initialize_cmd(self): """Create an executable command from script.""" diagnostics_root = DIAGNOSTICS.scripts script = self.script - script_file = (diagnostics_root / Path(script).expanduser()).absolute() + # Check if local diagnostic path exists + script_file = Path(script).expanduser().absolute() err_msg = f"Cannot execute script '{script}' ({script_file})" + if not script_file.is_file(): + logger.info( + "No local diagnostic script found. Attempting to load the script from the base repository." + ) + # Check if esmvaltool package is available + if importlib.util.find_spec("esmvaltool") is None: + logger.warning( + "The 'esmvaltool' package cannot be found. Please ensure it is installed." + ) + + # Try diagnostics_root + script_file = ( + diagnostics_root / Path(script).expanduser() + ).absolute() if not script_file.is_file(): raise DiagnosticError(f"{err_msg}: file does not exist.") From 691fce7f1365a4e88c8a60e1da13725b7ac109b6 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:33:51 +0100 Subject: [PATCH 05/17] Update changelog and version to v2.11.1 (#2590) --- CITATION.cff | 4 ++-- doc/changelog.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index f298987f64..c5a170d218 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -225,11 +225,11 @@ authors: orcid: "https://orcid.org/0009-0004-2333-3358" cff-version: 1.2.0 -date-released: 2024-07-03 +date-released: 2024-11-26 doi: "10.5281/zenodo.3387139" license: "Apache-2.0" message: "If you use this software, please cite it using these metadata." repository-code: "https://github.com/ESMValGroup/ESMValCore/" title: ESMValCore -version: "v2.11.0" +version: "v2.11.1" ... diff --git a/doc/changelog.rst b/doc/changelog.rst index 68c4fe1792..ee07934925 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -3,11 +3,46 @@ Changelog ========= +.. _changelog-v2-11-1: + +v2.11.1 +------- + +Highlights +~~~~~~~~~~ + +This is a bugfix release which enables lazy computations in more preprocessors +and allows installing the latests version of various dependencies, including +Iris (`v3.11.0 `__). + +This release includes + +Computational performance improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Optimize functions ``mask_landsea()``, ``mask_landseaice()`` and ``calculate_volume()`` for lazy input (:pull:`2515`) by :user:`schlunma` + +Installation +~~~~~~~~~~~~ + +- Remove support for Python 3.9 (:pull:`2447`) by :user:`valeriupredoi` +- Switch to new iris >= 3.10.0 API (:pull:`2500`) by :user:`schlunma` +- Pin dask to avoid 2024.8.0 - problems with masked fill/missing values (:pull:`2504`) by :user:`valeriupredoi` +- Fix rounding of Pandas datetimes in ICON CMORizer to allow installing latest Pandas version (:pull:`2529`) by :user:`valeriupredoi` + +Automatic testing +~~~~~~~~~~~~~~~~~ + +- Fix type hint for new mypy version (:pull:`2497`) by :user:`schlunma` +- Reformat datetime strings be in line with new ``isodate==0.7.0`` and actual ISO8601 and pin ``isodate>=0.7.0`` (:pull:`2546`) by :user:`valeriupredoi` + .. _changelog-v2-11-0: v2.11.0 ------- + Highlights +~~~~~~~~~~ - Performance improvements have been made to many preprocessors: @@ -189,7 +224,9 @@ Improvements v2.10.0 ------- + Highlights +~~~~~~~~~~ - All statistics preprocessors support the same operators and have a common :ref:`documentation `. In addition, arbitrary keyword arguments @@ -345,8 +382,10 @@ Improvements v2.9.0 ------ + Highlights ~~~~~~~~~~ + It is now possible to use the `Dask distributed scheduler `__, which can @@ -427,8 +466,10 @@ Preprocessor v2.8.1 ------ + Highlights ~~~~~~~~~~ + This release adds support for Python 3.11 and includes several bugfixes. This release includes: @@ -474,6 +515,7 @@ Automatic testing v2.8.0 ------ + Highlights ~~~~~~~~~~ @@ -674,6 +716,7 @@ Variable Derivation v2.7.1 ------ + Highlights ~~~~~~~~~~ @@ -702,6 +745,7 @@ Automatic testing v2.7.0 ------ + Highlights ~~~~~~~~~~ From a4b97376218f0f373017ebfa0c0a891086523aea Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 23:37:07 +0100 Subject: [PATCH 06/17] [pre-commit.ci] pre-commit autoupdate (#2586) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Bouwe Andela --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbc9b3117b..cea5c5c2cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.7.4" + rev: "v0.8.0" hooks: - id: ruff args: [--fix] From 1cf3862488e567573d7f9818937cfcf49b01a9fb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 29 Nov 2024 08:53:45 +0000 Subject: [PATCH 07/17] Support Python 3.13 (#2566) Co-authored-by: Bouwe Andela --- .github/workflows/build-and-deploy-on-pypi.yml | 4 ++-- .github/workflows/create-condalock-file.yml | 2 +- .github/workflows/install-from-conda.yml | 4 ++-- .github/workflows/install-from-condalock-file.yml | 2 +- .github/workflows/install-from-pypi.yml | 4 ++-- .github/workflows/install-from-source.yml | 4 ++-- .github/workflows/run-tests-monitor.yml | 4 ++-- .github/workflows/run-tests.yml | 4 ++-- pyproject.toml | 1 + 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-deploy-on-pypi.yml b/.github/workflows/build-and-deploy-on-pypi.yml index d3662b4eb2..ab70bd112d 100644 --- a/.github/workflows/build-and-deploy-on-pypi.yml +++ b/.github/workflows/build-and-deploy-on-pypi.yml @@ -22,10 +22,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: "3.13" - name: Install pep517 run: >- python -m diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index e234d709b2..97501f657c 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -27,7 +27,7 @@ jobs: with: auto-update-conda: true activate-environment: esmvaltool-fromlock - python-version: "3.12" + python-version: "3.13" miniforge-version: "latest" use-mamba: true - name: Update and show conda config diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml index d77ef193aa..9093dfedfd 100644 --- a/.github/workflows/install-from-conda.yml +++ b/.github/workflows/install-from-conda.yml @@ -39,7 +39,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] # fail-fast set to False allows all other tests # in the workflow to run regardless of any fail fail-fast: false @@ -74,7 +74,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] architecture: ["x64"] # need to force Intel, arm64 builds have issues fail-fast: false name: OSX Python ${{ matrix.python-version }} diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index 44a7839b55..86cce4fde7 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -29,7 +29,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: diff --git a/.github/workflows/install-from-pypi.yml b/.github/workflows/install-from-pypi.yml index bb18afa7c5..b89205b899 100644 --- a/.github/workflows/install-from-pypi.yml +++ b/.github/workflows/install-from-pypi.yml @@ -39,7 +39,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] # fail-fast set to False allows all other tests # in the workflow to run regardless of any fail fail-fast: false @@ -76,7 +76,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] architecture: ["x64"] # need to force Intel, arm64 builds have issues fail-fast: false name: OSX Python ${{ matrix.python-version }} diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml index 0d193bd79a..b8e48bdfe7 100644 --- a/.github/workflows/install-from-source.yml +++ b/.github/workflows/install-from-source.yml @@ -37,7 +37,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -73,7 +73,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] architecture: ["x64"] # need to force Intel, arm64 builds have issues fail-fast: false name: OSX Python ${{ matrix.python-version }} diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml index 27c0853730..97fefa3680 100644 --- a/.github/workflows/run-tests-monitor.yml +++ b/.github/workflows/run-tests-monitor.yml @@ -22,7 +22,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -54,7 +54,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] architecture: ["x64"] # need to force Intel, arm64 builds have issues fail-fast: false name: OSX Python ${{ matrix.python-version }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 743ac00f45..5a2d5c1a30 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -39,7 +39,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] fail-fast: false name: Linux Python ${{ matrix.python-version }} steps: @@ -74,7 +74,7 @@ jobs: runs-on: "macos-latest" strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] architecture: ["x64"] # need to force Intel, arm64 builds have issues fail-fast: false name: OSX Python ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index efbb3590c1..1c30bd95ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Atmospheric Science", "Topic :: Scientific/Engineering :: GIS", From d8ad2f07d29bb2eb1b38f2e7673a5f592eef47f0 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:23:41 +0100 Subject: [PATCH 08/17] Update `CFG` with configuration options given via command line (#2595) --- esmvalcore/_main.py | 4 ++-- tests/unit/main/test_esmvaltool.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 30e2668bdb..42fbc16092 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -439,9 +439,9 @@ def run(self, recipe, **kwargs): recipe = self._get_recipe(recipe) + CFG.update(kwargs) + CFG["resume_from"] = parse_resume(CFG["resume_from"], recipe) session = CFG.start_session(recipe.stem) - session.update(kwargs) - session["resume_from"] = parse_resume(session["resume_from"], recipe) self._run(recipe, session, cli_config_dir) diff --git a/tests/unit/main/test_esmvaltool.py b/tests/unit/main/test_esmvaltool.py index 7b9cb29662..1e03bbe5b1 100644 --- a/tests/unit/main/test_esmvaltool.py +++ b/tests/unit/main/test_esmvaltool.py @@ -21,12 +21,17 @@ @pytest.fixture def cfg(mocker, tmp_path): """Mock `esmvalcore.config.CFG`.""" - session = mocker.MagicMock() - cfg_dict = {"resume_from": []} - session.__getitem__.side_effect = cfg_dict.__getitem__ - session.__setitem__.side_effect = cfg_dict.__setitem__ - session.update.side_effect = cfg_dict.update + + cfg = mocker.MagicMock() + cfg.__getitem__.side_effect = cfg_dict.__getitem__ + cfg.__setitem__.side_effect = cfg_dict.__setitem__ + cfg.update.side_effect = cfg_dict.update + + session = mocker.MagicMock() + session.__getitem__.side_effect = cfg.__getitem__ + session.__setitem__.side_effect = cfg.__setitem__ + session.update.side_effect = cfg.update output_dir = tmp_path / "esmvaltool_output" session.session_dir = output_dir / "recipe_test" @@ -34,7 +39,6 @@ def cfg(mocker, tmp_path): session.preproc_dir = session.session_dir / "preproc_dir" session._fixed_file_dir = session.preproc_dir / "fixed_files" - cfg = mocker.Mock() cfg.start_session.return_value = session return cfg From 8b4a63afdf11a153c23f51bbe0c3a4fc856b0a71 Mon Sep 17 00:00:00 2001 From: Karen Garcia Perdomo <85649962+Karen-A-Garcia@users.noreply.github.com> Date: Fri, 29 Nov 2024 08:06:57 -0800 Subject: [PATCH 09/17] Missing 2m height coordinate and monotonicity for tasmin in CESM2 and CESM2-WACCM (#2574) Co-authored-by: Karen Garcia Perdomo Co-authored-by: Valeriu Predoi --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 187 +++++++++++++++- esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py | 4 + esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py | 12 + .../cmor/_fixes/cmip6/cesm2_waccm_fv2.py | 4 + .../cmor/_fixes/cmip6/test_cesm2.py | 207 +++++++++++++++++- .../cmor/_fixes/cmip6/test_cesm2_fv2.py | 15 ++ .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 41 ++++ .../cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py | 15 ++ 8 files changed, 476 insertions(+), 9 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index 0c5c0eed94..9b190adc63 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -2,6 +2,7 @@ from shutil import copyfile +import iris import numpy as np from netCDF4 import Dataset @@ -160,7 +161,7 @@ class Tas(Prw): def fix_metadata(self, cubes): """ - Add height (2m) coordinate. + Add height (2m) coordinate and time coordinate. Fix also done for prw. Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. @@ -172,15 +173,72 @@ def fix_metadata(self, cubes): Returns ------- - iris.cube.CubeList + iris.cube.CubeList, iris.cube.CubeList """ super().fix_metadata(cubes) # Specific code for tas cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube) - - return cubes + new_list = iris.cube.CubeList() + for cube in cubes: + try: + old_time = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + new_list.append(cube) + else: + if old_time.is_monotonic(): + new_list.append(cube) + else: + time_units = old_time.units + time_data = old_time.points + + # erase erroneously copy-pasted points + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + while len(idx_neg) > 0: + time_data = np.delete(time_data, idx_neg[0] + 1) + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + + # create the new time coord + new_time = iris.coords.DimCoord( + time_data, + standard_name="time", + var_name="time", + units=time_units, + ) + + # create a new cube with the right shape + dims = ( + time_data.shape[0], + cube.coord("latitude").shape[0], + cube.coord("longitude").shape[0], + ) + data = cube.data + new_data = np.ma.append( + data[: dims[0] - 1, :, :], data[-1, :, :] + ) + new_data = new_data.reshape(dims) + + tmp_cube = iris.cube.Cube( + new_data, + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=[ + (new_time, 0), + (cube.coord("latitude"), 1), + (cube.coord("longitude"), 2), + ], + ) + + new_list.append(tmp_cube) + + return new_list class Sftlf(Fix): @@ -286,3 +344,124 @@ def fix_metadata(self, cubes): if z_coord.standard_name is None: fix_ocean_depth_coord(cube) return cubes + + +class Pr(Fix): + """Fixes for pr.""" + + def fix_metadata(self, cubes): + """Fix time coordinates. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + """ + new_list = iris.cube.CubeList() + for cube in cubes: + try: + old_time = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + new_list.append(cube) + else: + if old_time.is_monotonic(): + new_list.append(cube) + else: + time_units = old_time.units + time_data = old_time.points + + # erase erroneously copy-pasted points + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + while len(idx_neg) > 0: + time_data = np.delete(time_data, idx_neg[0] + 1) + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + + # create the new time coord + new_time = iris.coords.DimCoord( + time_data, + standard_name="time", + var_name="time", + units=time_units, + ) + + # create a new cube with the right shape + dims = ( + time_data.shape[0], + cube.coord("latitude").shape[0], + cube.coord("longitude").shape[0], + ) + data = cube.data + new_data = np.ma.append( + data[: dims[0] - 1, :, :], data[-1, :, :] + ) + new_data = new_data.reshape(dims) + + tmp_cube = iris.cube.Cube( + new_data, + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=[ + (new_time, 0), + (cube.coord("latitude"), 1), + (cube.coord("longitude"), 2), + ], + ) + + new_list.append(tmp_cube) + return new_list + + +class Tasmin(Pr): + """Fixes for tasmin.""" + + def fix_metadata(self, cubes): + """Fix time and height 2m coordinates. + + Fix for time coming from Pr. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + add_scalar_height_coord(cube, height=2.0) + return cubes + + +class Tasmax(Pr): + """Fixes for tasmax.""" + + def fix_metadata(self, cubes): + """Fix time and height 2m coordinates. + + Fix for time coming from Pr. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + add_scalar_height_coord(cube, height=2.0) + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py index 4c55a20f03..2ce4b1073f 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py @@ -4,6 +4,7 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas Cl = BaseCl @@ -25,3 +26,6 @@ Tas = BaseTas + + +Pr = BasePr diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py index 156a656b5f..d3bbc4dafe 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py @@ -6,7 +6,10 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas +from .cesm2 import Tasmax as BaseTasmax +from .cesm2 import Tasmin as BaseTasmin class Cl(BaseCl): @@ -64,4 +67,13 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False): Siconc = SiconcFixScalarCoord +Pr = BasePr + + Tas = BaseTas + + +Tasmin = BaseTasmin + + +Tasmax = BaseTasmax diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py index 23f77fbd07..f888bb6244 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py @@ -3,6 +3,7 @@ from ..common import SiconcFixScalarCoord from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Omon as BaseOmon +from .cesm2 import Pr as BasePr from .cesm2 import Tas as BaseTas from .cesm2_waccm import Cl as BaseCl from .cesm2_waccm import Cli as BaseCli @@ -27,3 +28,6 @@ Tas = BaseTas + + +Pr = BasePr diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 5d504f6084..24df5db059 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -4,7 +4,9 @@ import unittest.mock import iris +import iris.cube import numpy as np +import pandas as pd import pytest from cf_units import Unit @@ -14,8 +16,11 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, + Tasmax, + Tasmin, Tos, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord @@ -202,7 +207,7 @@ def test_clw_fix(): def tas_cubes(): """Cubes to test fixes for ``tas``.""" time_coord = iris.coords.DimCoord( - [0.0, 1.0], + [0.0, 1.0, 2.0], var_name="time", standard_name="time", units="days since 1850-01-01 00:00:00", @@ -219,12 +224,12 @@ def tas_cubes(): (lon_coord, 2), ] ta_cube = iris.cube.Cube( - np.ones((2, 2, 2)), + np.ones((3, 2, 2)), var_name="ta", dim_coords_and_dims=coord_specs, ) tas_cube = iris.cube.Cube( - np.ones((2, 2, 2)), + np.ones((3, 2, 2)), var_name="tas", dim_coords_and_dims=coord_specs, ) @@ -336,17 +341,19 @@ def test_tas_fix_metadata(tas_cubes): with pytest.raises(iris.exceptions.CoordinateNotFoundError): cube.coord("height") height_coord = iris.coords.AuxCoord( - 2.0, + [2.0], var_name="height", standard_name="height", long_name="height", units=Unit("m"), attributes={"positive": "up"}, ) + vardef = get_var_info("CMIP6", "Amon", "tas") fix = Tas(vardef) out_cubes = fix.fix_metadata(tas_cubes) - assert out_cubes is tas_cubes + assert out_cubes[0] is tas_cubes[0] + assert out_cubes[1] is tas_cubes[1] for cube in out_cubes: assert cube.coord("longitude").has_bounds() assert cube.coord("latitude").has_bounds() @@ -357,6 +364,20 @@ def test_tas_fix_metadata(tas_cubes): with pytest.raises(iris.exceptions.CoordinateNotFoundError): cube.coord("height") + # de-monotonize time points + for cube in tas_cubes: + time = cube.coord("time") + points = np.array(time.points) + points[-1] = points[0] + dims = cube.coord_dims(time) + cube.remove_coord(time) + time = iris.coords.AuxCoord.from_coord(time) + cube.add_aux_coord(time.copy(points), dims) + + out_cubes = fix.fix_metadata(tas_cubes) + for cube in out_cubes: + assert cube.coord("time").is_monotonic() + def test_tos_fix_metadata(tos_cubes): """Test ``fix_metadata`` for ``tos``.""" @@ -420,3 +441,179 @@ def test_fgco2_fix_metadata(): def test_siconc_fix(): """Test fix for ``siconc``.""" assert Siconc is SiconcFixScalarCoord + + +@pytest.fixture +def pr_cubes(): + correct_time_coord = iris.coords.DimCoord( + points=[1.0, 2.0, 3.0, 4.0, 5.0], + var_name="time", + standard_name="time", + units="days since 1850-01-01", + ) + + lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + + lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + + correct_coord_specs = [ + (correct_time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + + correct_pr_cube = iris.cube.Cube( + np.ones((5, 1, 1)), + var_name="pr", + units="kg m-2 s-1", + dim_coords_and_dims=correct_coord_specs, + ) + + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + + return iris.cube.CubeList([correct_pr_cube, scalar_cube]) + + +def test_get_pr_fix(): + """Test pr fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix_metadata(pr_cubes): + """Test metadata fix.""" + vardef = get_var_info("CMIP6", "day", "pr") + fix = Pr(vardef) + + out_cubes = fix.fix_metadata(pr_cubes) + assert out_cubes[0].var_name == "pr" + coord = out_cubes[0].coord("time") + assert pd.Series(coord.points).is_monotonic_increasing + + # de-monotonize time points + for cube in pr_cubes: + if cube.var_name == "pr": + time = cube.coord("time") + points = np.array(time.points) + points[-1] = points[0] + dims = cube.coord_dims(time) + cube.remove_coord(time) + time = iris.coords.AuxCoord.from_coord(time) + cube.add_aux_coord(time.copy(points), dims) + + out_cubes = fix.fix_metadata(pr_cubes) + for cube in out_cubes: + if cube.var_name == "tas": + assert cube.coord("time").is_monotonic() + + +@pytest.fixture +def tasmin_cubes(): + correct_lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + wrong_lat_coord = iris.coords.DimCoord( + [0.0], var_name="latitudeCoord", standard_name="latitude" + ) + correct_lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + wrong_lon_coord = iris.coords.DimCoord( + [0.0], var_name="longitudeCoord", standard_name="longitude" + ) + correct_cube = iris.cube.Cube( + [[2.0]], + var_name="tasmin", + dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], + ) + wrong_cube = iris.cube.Cube( + [[2.0]], + var_name="ta", + dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], + ) + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +@pytest.fixture +def tasmax_cubes(): + correct_lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + wrong_lat_coord = iris.coords.DimCoord( + [0.0], var_name="latitudeCoord", standard_name="latitude" + ) + correct_lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + wrong_lon_coord = iris.coords.DimCoord( + [0.0], var_name="longitudeCoord", standard_name="longitude" + ) + correct_cube = iris.cube.Cube( + [[2.0]], + var_name="tasmax", + dim_coords_and_dims=[(correct_lat_coord, 0), (correct_lon_coord, 1)], + ) + wrong_cube = iris.cube.Cube( + [[2.0]], + var_name="ta", + dim_coords_and_dims=[(wrong_lat_coord, 0), (wrong_lon_coord, 1)], + ) + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + return iris.cube.CubeList([correct_cube, wrong_cube, scalar_cube]) + + +def test_get_tasmin_fix(): + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "tasmin") + assert fix == [Tasmin(None), GenericFix(None)] + + +def test_tasmin_fix_metadata(tasmin_cubes): + for cube in tasmin_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord("height") + height_coord = iris.coords.AuxCoord( + 2.0, + var_name="height", + standard_name="height", + long_name="height", + units=Unit("m"), + attributes={"positive": "up"}, + ) + vardef = get_var_info("CMIP6", "day", "tasmin") + fix = Tasmin(vardef) + + out_cubes = fix.fix_metadata(tasmin_cubes) + assert out_cubes[0].var_name == "tasmin" + coord = out_cubes[0].coord("height") + assert coord == height_coord + + +def test_get_tasmax_fix(): + fix = Fix.get_fixes("CMIP6", "CESM2", "day", "tasmax") + assert fix == [Tasmax(None), GenericFix(None)] + + +def test_tasmax_fix_metadata(tasmax_cubes): + for cube in tasmax_cubes: + with pytest.raises(iris.exceptions.CoordinateNotFoundError): + cube.coord("height") + height_coord = iris.coords.AuxCoord( + 2.0, + var_name="height", + standard_name="height", + long_name="height", + units=Unit("m"), + attributes={"positive": "up"}, + ) + vardef = get_var_info("CMIP6", "day", "tasmax") + fix = Tasmax(vardef) + + out_cubes = fix.fix_metadata(tasmax_cubes) + assert out_cubes[0].var_name == "tasmax" + coord = out_cubes[0].coord("height") + assert coord == height_coord diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py index 50e67e5d0f..89bc345bbb 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py @@ -2,6 +2,7 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_fv2 import ( Cl, @@ -9,6 +10,7 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, ) @@ -76,8 +78,21 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-FV2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index 363bf0d80c..8e4409542f 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -9,15 +9,21 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas +from esmvalcore.cmor._fixes.cmip6.cesm2 import Tasmax as BaseTasmax +from esmvalcore.cmor._fixes.cmip6.cesm2 import Tasmin as BaseTasmin from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import ( Cl, Cli, Clw, Fgco2, Omon, + Pr, Siconc, Tas, + Tasmax, + Tasmin, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord from esmvalcore.cmor._fixes.fix import GenericFix @@ -119,8 +125,43 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr + + +def test_get_tasmin_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tasmin") + assert fix == [Tasmin(None), GenericFix(None)] + + +def test_tasmin_fix(): + """Test fix for ``Tasmin``.""" + assert Tasmin is BaseTasmin + + +def test_get_tasmax_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM", "day", "tasmax") + assert fix == [Tasmax(None), GenericFix(None)] + + +def test_tasmax_fix(): + """Test fix for ``Tasmax``.""" + assert Tasmax is BaseTasmax diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py index e61fec5745..6d115234f2 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py @@ -1,6 +1,7 @@ """Tests for the fixes of CESM2-WACCM-FV2.""" from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Pr as BasePr from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2_waccm_fv2 import ( @@ -9,6 +10,7 @@ Clw, Fgco2, Omon, + Pr, Siconc, Tas, ) @@ -76,8 +78,21 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes("CMIP6", "CESM2-WACCM-FV2", "Amon", "tas") assert fix == [Tas(None), GenericFix(None)] + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM-FV2", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): """Test fix for ``tas``.""" assert Tas is BaseTas + + +def test_get_pr_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes("CMIP6", "CESM2-WACCM_FV2", "day", "pr") + assert fix == [Pr(None), GenericFix(None)] + + +def test_pr_fix(): + """Test fix for ``Pr``.""" + assert Pr is BasePr From 11a478781037d8d31b0a583f4cb4baa0f8341547 Mon Sep 17 00:00:00 2001 From: Romain Beucher Date: Mon, 2 Dec 2024 22:55:38 +1000 Subject: [PATCH 10/17] Fix 2593 Change log INFO to DEBUG (#2600) --- esmvalcore/_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 66018c2789..27a6b83d14 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -392,7 +392,7 @@ def _initialize_cmd(self): script_file = Path(script).expanduser().absolute() err_msg = f"Cannot execute script '{script}' ({script_file})" if not script_file.is_file(): - logger.info( + logger.debug( "No local diagnostic script found. Attempting to load the script from the base repository." ) # Check if esmvaltool package is available From 3a57194dc5a6949f4418bc31a5311dc4eaa73acb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 3 Dec 2024 12:23:09 +0100 Subject: [PATCH 11/17] Save all files in a task at the same time to avoid recomputing intermediate results (#2522) Co-authored-by: Valeriu Predoi --- doc/quickstart/configure.rst | 54 +++- environment.yml | 1 + esmvalcore/_recipe/recipe.py | 7 +- esmvalcore/config/_config_validators.py | 1 + .../configurations/defaults/logging.yml | 2 + esmvalcore/preprocessor/__init__.py | 29 ++- esmvalcore/preprocessor/_dask_progress.py | 242 ++++++++++++++++++ esmvalcore/preprocessor/_io.py | 61 +++-- pyproject.toml | 1 + .../integration/preprocessor/_io/test_save.py | 65 +++-- .../preprocessor/test_preprocessing_task.py | 2 +- tests/integration/recipe/test_recipe.py | 2 + tests/unit/config/test_config.py | 1 + tests/unit/preprocessor/test_dask_progress.py | 51 ++++ .../preprocessor/test_preprocessor_file.py | 1 + tests/unit/recipe/test_recipe.py | 16 +- 16 files changed, 480 insertions(+), 56 deletions(-) create mode 100644 esmvalcore/config/configurations/defaults/logging.yml create mode 100644 esmvalcore/preprocessor/_dask_progress.py create mode 100644 tests/unit/preprocessor/test_dask_progress.py diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index c65fdbd1c5..baf4dd2998 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -129,8 +129,8 @@ More information about this can be found :ref:`here `. .. _config_options: -Configuration options -===================== +Top level configuration options +=============================== Note: the following entries use Python syntax. For example, Python's ``None`` is YAML's ``null``, Python's ``True`` is YAML's @@ -170,6 +170,8 @@ For example, Python's ``None`` is YAML's ``null``, Python's ``True`` is YAML's | ``log_level`` | Log level of the console (``debug``, | :obj:`str` | ``info`` | | | ``info``, ``warning``, ``error``) | | | +-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``logging`` | :ref:`config-logging` | :obj:`dict` | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ | ``max_datasets`` | Maximum number of datasets to use, see | :obj:`int` | ``None`` (all datasets from recipe) | | | :ref:`running` | | | +-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ @@ -269,6 +271,54 @@ For example, Python's ``None`` is YAML's ``null``, Python's ``True`` is YAML's will be downloaded; otherwise, local data will be used. +.. _config-logging: + +Logging configuration +===================== + +Configure what information is logged and how it is presented in the ``logging`` +section. + +.. note:: + + Not all logging configuration is available here yet, see :issue:`2596`. + +Configuration file example: + +.. code:: yaml + + logging: + log_progress_interval: 10s + +will log progress of Dask computations every 10 seconds instead of showing a +progress bar. + +Command line example: + +.. code:: bash + + esmvaltool run --logging='{"log_progress_interval": "1m"}' recipe_example.yml + + +will log progress of Dask computations every minute instead of showing a +progress bar. + +Available options: + ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| Option | Description | Type | Default value | ++===============================+========================================+=============================+========================================+ +| ``log_progress_interval`` | When running computations with Dask, | :obj:`str` or :obj:`float` | 0 | +| | log progress every | | | +| | ``log_progress_interval`` instead of | | | +| | showing a progress bar. The value can | | | +| | be specified in the format accepted by | | | +| | :func:`dask.utils.parse_timedelta`. A | | | +| | negative value disables any progress | | | +| | reporting. A progress bar is only | | | +| | shown if ``max_parallel_tasks: 1``. | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ + .. _config-dask: Dask configuration diff --git a/environment.yml b/environment.yml index 7a4b6e2201..321b0484c5 100644 --- a/environment.yml +++ b/environment.yml @@ -40,6 +40,7 @@ dependencies: - python-stratify >=0.3 - pyyaml - requests + - rich - scipy >=1.6 - shapely >=2.0.0 - yamale diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 55e789d6f4..8d4809ffa0 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -220,7 +220,10 @@ def _get_default_settings(dataset): settings["remove_supplementary_variables"] = {} # Configure saving cubes to file - settings["save"] = {"compress": session["compress_netcdf"]} + settings["save"] = { + "compress": session["compress_netcdf"], + "compute": False, + } if facets["short_name"] != facets["original_short_name"]: settings["save"]["alias"] = facets["short_name"] @@ -381,6 +384,8 @@ def _get_downstream_settings(step, order, products): if key in remaining_steps: if all(p.settings.get(key, object()) == value for p in products): settings[key] = value + # Set the compute argument to the save step. + settings["save"] = {"compute": some_product.settings["save"]["compute"]} return settings diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 0722b346b5..b12ed08204 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -332,6 +332,7 @@ def validate_extra_facets_dir(value): "exit_on_warning": validate_bool, "extra_facets_dir": validate_extra_facets_dir, "log_level": validate_string, + "logging": validate_dict, "max_datasets": validate_int_positive_or_none, "max_parallel_tasks": validate_int_or_none, "max_years": validate_int_positive_or_none, diff --git a/esmvalcore/config/configurations/defaults/logging.yml b/esmvalcore/config/configurations/defaults/logging.yml new file mode 100644 index 0000000000..d1cd1948f2 --- /dev/null +++ b/esmvalcore/config/configurations/defaults/logging.yml @@ -0,0 +1,2 @@ +logging: + log_progress_interval: 0. diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 851aae49f0..2c956aa0ad 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -9,6 +9,7 @@ from pprint import pformat from typing import Any, Iterable +from dask.delayed import Delayed from iris.cube import Cube from .._provenance import TrackedFile @@ -25,6 +26,7 @@ ) from ._compare_with_refs import bias, distance_metric from ._cycles import amplitude +from ._dask_progress import _compute_with_progress from ._derive import derive from ._detrend import detrend from ._io import ( @@ -428,6 +430,9 @@ def preprocess( ) ) + if step == "save": + return result + items = [] for item in result: if isinstance(item, (PreprocessorFile, Cube, str, Path)): @@ -536,22 +541,24 @@ def cubes(self): def cubes(self, value): self._cubes = value - def save(self): + def save(self) -> Delayed | None: """Save cubes to disk.""" - preprocess( + return preprocess( self._cubes, "save", input_files=self._input_files, **self.settings["save"], - ) + )[0] - def close(self): + def close(self) -> Delayed | None: """Close the file.""" + result = None if self._cubes is not None: self._update_attributes() - self.save() + result = self.save() self._cubes = None self.save_provenance() + return result def _update_attributes(self): """Update product attributes from cube metadata.""" @@ -693,7 +700,7 @@ def _initialize_products(self, products): for product in products: product.initialize_provenance(self.activity) - def _run(self, _): + def _run(self, _) -> list[str]: """Run the preprocessor.""" self._initialize_product_provenance() @@ -703,6 +710,7 @@ def _run(self, _): blocks = get_step_blocks(steps, self.order) saved = set() + delayeds = [] for block in blocks: logger.debug("Running block %s", block) if block[0] in MULTI_MODEL_FUNCTIONS: @@ -718,14 +726,19 @@ def _run(self, _): product.apply(step, self.debug) if block == blocks[-1]: product.cubes # noqa: B018 pylint: disable=pointless-statement - product.close() + delayed = product.close() + delayeds.append(delayed) saved.add(product.filename) for product in self.products: if product.filename not in saved: product.cubes # noqa: B018 pylint: disable=pointless-statement - product.close() + delayed = product.close() + delayeds.append(delayed) + logger.info("Computing and saving data for task %s", self.name) + delayeds = [d for d in delayeds if d is not None] + _compute_with_progress(delayeds, description=self.name) metadata_files = write_metadata( self.products, self.write_ncl_interface ) diff --git a/esmvalcore/preprocessor/_dask_progress.py b/esmvalcore/preprocessor/_dask_progress.py new file mode 100644 index 0000000000..bcfc3380d5 --- /dev/null +++ b/esmvalcore/preprocessor/_dask_progress.py @@ -0,0 +1,242 @@ +"""Progress bars for use with Dask.""" + +from __future__ import annotations + +import contextlib +import datetime +import logging +import threading +import time +from collections.abc import Iterable + +import dask.diagnostics +import distributed +import rich.progress +from dask.delayed import Delayed + +from esmvalcore.config import CFG + +logger = logging.getLogger(__name__) + + +class RichProgressBar(dask.diagnostics.Callback): + """Progress bar using `rich` for the Dask default scheduler.""" + + # Disable warnings about design choices that have been made in the base class. + # pylint: disable=method-hidden,super-init-not-called,too-few-public-methods,unused-argument,useless-suppression + + # Adapted from https://github.com/dask/dask/blob/0f3e5ff6e642e7661b3f855bfd192a6f6fb83b49/dask/diagnostics/progress.py#L32-L153 + def __init__(self): + self.progress = rich.progress.Progress( + rich.progress.TaskProgressColumn(), + rich.progress.BarColumn(bar_width=80), + rich.progress.MofNCompleteColumn(), + rich.progress.TimeElapsedColumn(), + redirect_stdout=False, + redirect_stderr=False, + ) + self.task = self.progress.add_task(description="progress") + self._dt = 0.1 + self._state = None + self._running = False + self._timer = None + + def _start(self, dsk): + self._state = None + # Start background thread + self._running = True + self._timer = threading.Thread(target=self._timer_func) + self._timer.daemon = True + self._timer.start() + + def _start_state(self, dsk, state): + self.progress.start() + total = sum( + len(state[k]) for k in ["ready", "waiting", "running", "finished"] + ) + self.progress.update(self.task, total=total) + + def _pretask(self, key, dsk, state): + self._state = state + + def _finish(self, dsk, state, errored): + self._running = False + self._timer.join() + self._draw_bar() + self.progress.stop() + + def _timer_func(self): + """Background thread for updating the progress bar.""" + while self._running: + self._draw_bar() + time.sleep(self._dt) + + def _draw_bar(self): + state = self._state + completed = len(state["finished"]) if state else 0 + self.progress.update(self.task, completed=completed) + + +class RichDistributedProgressBar( + distributed.diagnostics.progressbar.TextProgressBar +): + """Progress bar using `rich` for the Dask distributed scheduler.""" + + # Disable warnings about design choices that have been made in the base class. + # pylint: disable=too-few-public-methods,unused-argument,useless-suppression + + def __init__(self, keys) -> None: + self.progress = rich.progress.Progress( + rich.progress.TaskProgressColumn(), + rich.progress.BarColumn(bar_width=80), + rich.progress.MofNCompleteColumn(), + rich.progress.TimeElapsedColumn(), + redirect_stdout=False, + redirect_stderr=False, + ) + self.progress.start() + self.task_id = self.progress.add_task(description="progress") + super().__init__(keys) + + def _draw_bar(self, remaining, all, **kwargs): # pylint: disable=redefined-builtin + completed = all - remaining + self.progress.update(self.task_id, completed=completed, total=all) + + def _draw_stop(self, **kwargs): + if kwargs.get("status") == "finished": + self._draw_bar(remaining=0, all=self.progress.tasks[0].total) + self.progress.stop() + + +class ProgressLogger(dask.diagnostics.ProgressBar): + """Progress logger for the Dask default scheduler.""" + + # Disable warnings about design choices that have been made in the base class. + # pylint: disable=too-few-public-methods,unused-argument,useless-suppression + + def __init__( + self, + log_interval: str | float = "1s", + description: str = "", + ) -> None: + self._desc = f"{description} " if description else description + self._log_interval = dask.utils.parse_timedelta( + log_interval, default="s" + ) + self._prev_elapsed = 0.0 + interval = dask.utils.parse_timedelta("1s", default="s") + super().__init__(dt=interval) + self._file = None + + def _draw_bar(self, frac: float, elapsed: float) -> None: + if (elapsed - self._prev_elapsed) < self._log_interval and frac != 1.0: + return + self._prev_elapsed = elapsed + pbar = "#" * int(self._width * frac) + percent = int(100 * frac) + elapsed_fmt = dask.utils.format_time(elapsed) + desc_width = 30 + msg = ( + f"{self._desc:<{desc_width}}[{pbar:<{self._width}}] | " + f"{percent:3}% Completed | {elapsed_fmt}" + ) + logger.info(msg) + + +class DistributedProgressLogger( + distributed.diagnostics.progressbar.TextProgressBar +): + """Progress logger for the Dask distributed scheduler.""" + + # Disable warnings about design choices that have been made in the base class. + # pylint: disable=too-few-public-methods,unused-argument,useless-suppression + + def __init__( + self, + keys, + log_interval: str | float = "1s", + description: str = "", + ) -> None: + self._desc = f"{description} " if description else description + self._log_interval = dask.utils.parse_timedelta( + log_interval, default="s" + ) + self._prev_elapsed = 0.0 + super().__init__(keys, interval="1s") + + def _draw_bar( + self, + remaining: int, + all: int, # pylint: disable=redefined-builtin + **kwargs, + ) -> None: + frac = (1 - remaining / all) if all else 1.0 + if ( + self.elapsed - self._prev_elapsed + ) < self._log_interval and frac != 1.0: + return + self._prev_elapsed = self.elapsed + pbar = "#" * int(self.width * frac) + percent = int(100 * frac) + elapsed = dask.utils.format_time(self.elapsed) + desc_width = 30 + msg = ( + f"{self._desc:<{desc_width}}[{pbar:<{self.width}}] | " + f"{percent:3}% Completed | {elapsed}" + ) + logger.info(msg) + + def _draw_stop(self, **kwargs): + pass + + +def _compute_with_progress( + delayeds: Iterable[Delayed], + description: str, +) -> None: + """Compute delayeds while displaying a progress bar.""" + use_distributed = True + try: + distributed.get_client() + except ValueError: + use_distributed = False + + log_progress_interval = CFG["logging"]["log_progress_interval"] + if isinstance(log_progress_interval, (str, datetime.timedelta)): + log_progress_interval = dask.utils.parse_timedelta( + log_progress_interval + ) + + if CFG["max_parallel_tasks"] != 1 and log_progress_interval == 0.0: + # Enable progress logging if `max_parallel_tasks` > 1 to avoid clutter. + log_progress_interval = 10.0 + + # There are three possible options, depending on the value of + # CFG["log_progress_interval"]: + # < 0: no progress reporting + # = 0: show progress bar + # > 0: log progress at this interval + if log_progress_interval < 0.0: + dask.compute(delayeds) + elif use_distributed: + futures = dask.persist(delayeds) + futures = distributed.client.futures_of(futures) + if log_progress_interval == 0.0: + RichDistributedProgressBar(futures) + else: + DistributedProgressLogger( + futures, + log_interval=log_progress_interval, + description=description, + ) + dask.compute(futures) + else: + if log_progress_interval == 0.0: + ctx: contextlib.AbstractContextManager = RichProgressBar() + else: + ctx = ProgressLogger( + description=description, + log_interval=log_progress_interval, + ) + with ctx: + dask.compute(delayeds) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 83f4d9bae5..eccac411f2 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -5,6 +5,7 @@ import copy import logging import os +from collections.abc import Sequence from itertools import groupby from pathlib import Path from typing import NamedTuple, Optional @@ -17,6 +18,7 @@ import numpy as np import yaml from cf_units import suppress_errors +from dask.delayed import Delayed from iris.cube import CubeList from esmvalcore.cmor.check import CheckLevels @@ -405,19 +407,25 @@ def concatenate(cubes, check_level=CheckLevels.DEFAULT): def save( - cubes, filename, optimize_access="", compress=False, alias="", **kwargs -): + cubes: Sequence[iris.cube.Cube], + filename: Path | str, + optimize_access: str = "", + compress: bool = False, + alias: str = "", + compute: bool = True, + **kwargs, +) -> Delayed | None: """Save iris cubes to file. Parameters ---------- - cubes: iterable of iris.cube.Cube + cubes: Data cubes to be saved - filename: str + filename: Name of target file - optimize_access: str + optimize_access: Set internal NetCDF chunking to favour a reading scheme Values can be map or timeseries, which improve performance when @@ -426,16 +434,30 @@ def save( case the better performance will be avhieved by loading all the values in that coordinate at a time - compress: bool, optional + compress: Use NetCDF internal compression. - alias: str, optional + alias: Var name to use when saving instead of the one in the cube. + compute : bool, default=True + Default is ``True``, meaning complete the file immediately, and return ``None``. + + When ``False``, create the output file but don't write any lazy array content to + its variables, such as lazy cube data or aux-coord points and bounds. + Instead return a :class:`dask.delayed.Delayed` which, when computed, will + stream all the lazy content via :meth:`dask.store`, to complete the file. + Several such data saves can be performed in parallel, by passing a list of them + into a :func:`dask.compute` call. + + **kwargs: + See :func:`iris.fileformats.netcdf.saver.save` for additional + keyword arguments. + Returns ------- - str - filename + :class:`dask.delayed.Delayed` or :obj:`None` + A delayed object that can be used to save the data in the cube. Raises ------ @@ -445,6 +467,9 @@ def save( if not cubes: raise ValueError(f"Cannot save empty cubes '{cubes}'") + if Path(filename).suffix.lower() == ".nc": + kwargs["compute"] = compute + # Rename some arguments kwargs["target"] = filename kwargs["zlib"] = compress @@ -462,7 +487,7 @@ def save( cubes, filename, ) - return filename + return None for cube in cubes: logger.debug( @@ -480,13 +505,11 @@ def save( elif optimize_access == "timeseries": dims = set(cube.coord_dims("time")) else: - dims = tuple() - for coord_dims in ( - cube.coord_dims(dimension) - for dimension in optimize_access.split(" ") - ): - dims += coord_dims - dims = set(dims) + dims = { + dim + for coord_name in optimize_access.split(" ") + for dim in cube.coord_dims(coord_name) + } kwargs["chunksizes"] = tuple( length if index in dims else 1 @@ -512,9 +535,7 @@ def save( category=UserWarning, module="iris", ) - iris.save(cubes, **kwargs) - - return filename + return iris.save(cubes, **kwargs) def _get_debug_filename(filename, step): diff --git a/pyproject.toml b/pyproject.toml index 1c30bd95ee..61de91ae25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ dependencies = [ "pybtex", "pyyaml", "requests", + "rich", "scipy>=1.6", "scitools-iris>=3.11", # 3.11 first to support Numpy 2 and Python 3.13 "shapely>=2.0.0", diff --git a/tests/integration/preprocessor/_io/test_save.py b/tests/integration/preprocessor/_io/test_save.py index 0e4f6b4366..20278fb155 100644 --- a/tests/integration/preprocessor/_io/test_save.py +++ b/tests/integration/preprocessor/_io/test_save.py @@ -1,9 +1,13 @@ """Integration tests for :func:`esmvalcore.preprocessor.save`.""" +import logging +import re + import iris import netCDF4 import numpy as np import pytest +from dask.delayed import Delayed from iris.coords import DimCoord from iris.cube import Cube, CubeList @@ -59,32 +63,51 @@ def _check_chunks(path, expected_chunks): def test_save(cube, filename): """Test save.""" - path = save([cube], filename) - loaded_cube = iris.load_cube(path) + delayed = save([cube], filename) + assert delayed is None + loaded_cube = iris.load_cube(filename) + _compare_cubes(cube, loaded_cube) + + +def test_delayed_save(cube, filename): + """Test save.""" + delayed = save([cube], filename, compute=False) + assert isinstance(delayed, Delayed) + delayed.compute() + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) +def test_save_noop(cube, filename, caplog): + """Test save.""" + cube.data = cube.lazy_data() + save([cube], filename) + with caplog.at_level(logging.DEBUG): + save([cube], filename) + assert re.findall("Not saving cubes .* to avoid data loss.", caplog.text) + + def test_save_create_parent_dir(cube, tmp_path): filename = tmp_path / "preproc" / "something" / "test.nc" - path = save([cube], filename) - loaded_cube = iris.load_cube(path) + save([cube], filename) + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) def test_save_alias(cube, filename): """Test save.""" - path = save([cube], filename, alias="alias") - loaded_cube = iris.load_cube(path) + save([cube], filename, alias="alias") + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) assert loaded_cube.var_name == "alias" def test_save_zlib(cube, filename): """Test save.""" - path = save([cube], filename, compress=True) - loaded_cube = iris.load_cube(path) + save([cube], filename, compress=True) + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) - with netCDF4.Dataset(path, "r") as handler: + with netCDF4.Dataset(filename, "r") as handler: sample_filters = handler.variables["sample"].filters() assert sample_filters["zlib"] is True assert sample_filters["shuffle"] is True @@ -106,32 +129,32 @@ def test_fail_without_filename(cube): def test_save_optimized_map(cube, filename): """Test save.""" - path = save([cube], filename, optimize_access="map") - loaded_cube = iris.load_cube(path) + save([cube], filename, optimize_access="map") + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) - _check_chunks(path, [2, 2, 1]) + _check_chunks(filename, [2, 2, 1]) def test_save_optimized_timeseries(cube, filename): """Test save.""" - path = save([cube], filename, optimize_access="timeseries") - loaded_cube = iris.load_cube(path) + save([cube], filename, optimize_access="timeseries") + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) - _check_chunks(path, [1, 1, 2]) + _check_chunks(filename, [1, 1, 2]) def test_save_optimized_lat(cube, filename): """Test save.""" - path = save([cube], filename, optimize_access="latitude") - loaded_cube = iris.load_cube(path) + save([cube], filename, optimize_access="latitude") + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) expected_chunks = [2, 1, 1] - _check_chunks(path, expected_chunks) + _check_chunks(filename, expected_chunks) def test_save_optimized_lon_time(cube, filename): """Test save.""" - path = save([cube], filename, optimize_access="longitude time") - loaded_cube = iris.load_cube(path) + save([cube], filename, optimize_access="longitude time") + loaded_cube = iris.load_cube(filename) _compare_cubes(cube, loaded_cube) - _check_chunks(path, [1, 2, 2]) + _check_chunks(filename, [1, 2, 2]) diff --git a/tests/integration/preprocessor/test_preprocessing_task.py b/tests/integration/preprocessor/test_preprocessing_task.py index 6b3023f1d2..5b74a94cda 100644 --- a/tests/integration/preprocessor/test_preprocessing_task.py +++ b/tests/integration/preprocessor/test_preprocessing_task.py @@ -24,7 +24,7 @@ def test_load_save_task(tmp_path): [ PreprocessorFile( filename=tmp_path / "tas_out.nc", - settings={}, + settings={"save": {"compute": False}}, datasets=[dataset], ), ] diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index f486db1657..90b4985a6e 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -110,6 +110,7 @@ def _get_default_settings_for_chl(save_filename): "save": { "compress": False, "filename": save_filename, + "compute": False, }, } return defaults @@ -693,6 +694,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, session): "save": { "compress": False, "filename": product.filename, + "compute": False, }, } assert product.settings == defaults diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 194724a317..44b8d6ee3e 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -191,6 +191,7 @@ def test_load_default_config(cfg_default, monkeypatch): "exit_on_warning": False, "extra_facets_dir": [], "log_level": "info", + "logging": {"log_progress_interval": 0.0}, "max_datasets": None, "max_parallel_tasks": None, "max_years": None, diff --git a/tests/unit/preprocessor/test_dask_progress.py b/tests/unit/preprocessor/test_dask_progress.py new file mode 100644 index 0000000000..6712272386 --- /dev/null +++ b/tests/unit/preprocessor/test_dask_progress.py @@ -0,0 +1,51 @@ +"""Test :mod:`esmvalcore.preprocessor._dask_progress`.""" + +import logging +import time + +import dask +import distributed +import pytest + +from esmvalcore.preprocessor import _dask_progress + + +@pytest.mark.parametrize("use_distributed", [False, True]) +@pytest.mark.parametrize("interval", [-1, 0.0, 0.2]) +def test_compute_with_progress( + capsys, + caplog, + monkeypatch, + use_distributed, + interval, +): + caplog.set_level(logging.INFO) + if use_distributed: + client = distributed.Client(n_workers=1, threads_per_worker=1) + else: + client = None + + monkeypatch.setitem(_dask_progress.CFG, "max_parallel_tasks", 1) + monkeypatch.setitem( + _dask_progress.CFG["logging"], + "log_progress_interval", + f"{interval}s" if interval > 0 else interval, + ) + + def func(delay: float) -> None: + time.sleep(delay) + + delayeds = [dask.delayed(func)(0.11)] + _dask_progress._compute_with_progress(delayeds, description="test") + if interval == 0.0: + # Assert that some progress bar has been written to stdout. + progressbar = capsys.readouterr().out + else: + # Assert that some progress bar has been logged. + progressbar = caplog.text + if interval < 0.0: + assert not progressbar + else: + assert "100%" in progressbar + if client is not None: + client.shutdown() diff --git a/tests/unit/preprocessor/test_preprocessor_file.py b/tests/unit/preprocessor/test_preprocessor_file.py index d386dbc1e6..6e845e8aea 100644 --- a/tests/unit/preprocessor/test_preprocessor_file.py +++ b/tests/unit/preprocessor/test_preprocessor_file.py @@ -162,4 +162,5 @@ def test_save(mock_preprocess): mock.call( mock.sentinel.cubes, "save", input_files=mock.sentinel.input_files ), + mock.call().__getitem__(0), ] diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index 9934f02d3b..5acc625c8d 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -243,7 +243,10 @@ def test_multi_model_filename_full(): def test_update_multiproduct_multi_model_statistics(): """Test ``_update_multiproduct``.""" - settings = {"multi_model_statistics": {"statistics": ["mean", "std_dev"]}} + settings = { + "multi_model_statistics": {"statistics": ["mean", "std_dev"]}, + "save": {"compute": False}, + } common_attributes = { "project": "CMIP6", "diagnostic": "d", @@ -358,6 +361,7 @@ def test_update_multiproduct_multi_model_statistics_percentile(): {"operator": "percentile", "percent": 95.0}, ] }, + "save": {"compute": False}, } common_attributes = { "project": "CMIP6", @@ -468,7 +472,8 @@ def test_update_multiproduct_multi_model_statistics_percentile(): def test_update_multiproduct_ensemble_statistics(): """Test ``_update_multiproduct``.""" settings = { - "ensemble_statistics": {"statistics": ["median"], "span": "full"} + "ensemble_statistics": {"statistics": ["median"], "span": "full"}, + "save": {"compute": False}, } common_attributes = { "dataset": "CanESM2", @@ -539,6 +544,7 @@ def test_update_multiproduct_ensemble_statistics_percentile(): ], "span": "full", }, + "save": {"compute": False}, } common_attributes = { @@ -773,7 +779,11 @@ def test_get_default_settings(mocker): settings = _recipe._get_default_settings(dataset) assert settings == { "remove_supplementary_variables": {}, - "save": {"compress": False, "alias": "sic"}, + "save": { + "compress": False, + "alias": "sic", + "compute": False, + }, } From 0f9e8884ef4d6c0ba8e5fb78b4988ba0f26a9185 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:46:53 +0000 Subject: [PATCH 12/17] [pre-commit.ci] pre-commit autoupdate (#2604) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cea5c5c2cc..8b7004b93d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.8.0" + rev: "v0.8.1" hooks: - id: ruff args: [--fix] From 4c36a0c7be2a48b32e1394e6e3212c9f7913a2f7 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 3 Dec 2024 18:18:48 +0100 Subject: [PATCH 13/17] Avoid a crash when there is a timeout when shutting down the Dask cluster (#2580) --- esmvalcore/config/_dask.py | 9 ++++++++- tests/unit/config/test_dask.py | 7 +++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/esmvalcore/config/_dask.py b/esmvalcore/config/_dask.py index effd33058f..4de51e4aef 100644 --- a/esmvalcore/config/_dask.py +++ b/esmvalcore/config/_dask.py @@ -80,4 +80,11 @@ def get_distributed_client(): if client is not None: client.close() if cluster is not None: - cluster.close() + try: + cluster.close() + except TimeoutError: + logger.warning( + "Timeout while trying to shut down the cluster at %s, " + "you may want to check it was stopped.", + cluster.scheduler_address, + ) diff --git a/tests/unit/config/test_dask.py b/tests/unit/config/test_dask.py index e965c90a2e..8efc305023 100644 --- a/tests/unit/config/test_dask.py +++ b/tests/unit/config/test_dask.py @@ -37,7 +37,8 @@ def test_get_distributed_client_external(mocker, tmp_path, warn_unused_args): mock_client.close.assert_called() -def test_get_distributed_client_slurm(mocker, tmp_path): +@pytest.mark.parametrize("shutdown_timeout", [False, True]) +def test_get_distributed_client_slurm(mocker, tmp_path, shutdown_timeout): cfg = { "cluster": { "type": "dask_jobqueue.SLURMCluster", @@ -66,10 +67,12 @@ def test_get_distributed_client_slurm(mocker, tmp_path): create_autospec=True, return_value=mock_module, ) + mock_cluster = mock_cluster_cls.return_value + if shutdown_timeout: + mock_cluster.close.side_effect = TimeoutError with _dask.get_distributed_client() as client: assert client is mock_client mock_client.close.assert_called() - mock_cluster = mock_cluster_cls.return_value _dask.Client.assert_called_with(address=mock_cluster.scheduler_address) args = {k: v for k, v in cfg["cluster"].items() if k != "type"} mock_cluster_cls.assert_called_with(**args) From 65c7b28e883d24eb2471cf34a215553b8895a3c5 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:41:49 +0100 Subject: [PATCH 14/17] Add support for native ERA5 data in GRIB format (#2178) Co-authored-by: Valeriu Predoi Co-authored-by: Bouwe Andela Co-authored-by: Bettina Gier --- doc/quickstart/configure.rst | 2 +- doc/quickstart/find_data.rst | 109 ++- esmvalcore/_provenance.py | 8 +- esmvalcore/_recipe/recipe.py | 30 + esmvalcore/cmor/_fixes/fix.py | 2 + esmvalcore/cmor/_fixes/native6/era5.py | 186 ++++- esmvalcore/config-developer.yml | 2 + .../configurations/defaults/config-user.yml | 4 +- .../config/extra_facets/native6-era5.yml | 196 ++++++ esmvalcore/preprocessor/_io.py | 9 +- pyproject.toml | 2 +- .../cmor/_fixes/native6/test_era5.py | 666 ++++++++++++++---- tests/integration/cmor/test_fix.py | 30 +- tests/integration/conftest.py | 38 +- .../integration/preprocessor/_io/test_load.py | 20 + tests/integration/recipe/test_recipe.py | 113 +++ tests/sample_data/iris-sample-data/LICENSE | 10 + .../iris-sample-data/polar_stereo.grib2 | Bin 0 -> 25934 bytes tests/unit/provenance/test_trackedfile.py | 66 +- 19 files changed, 1296 insertions(+), 197 deletions(-) create mode 100644 esmvalcore/config/extra_facets/native6-era5.yml create mode 100644 tests/sample_data/iris-sample-data/LICENSE create mode 100644 tests/sample_data/iris-sample-data/polar_stereo.grib2 diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index baf4dd2998..78ce5dcea2 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -974,7 +974,7 @@ infrastructure. The following example illustrates the concept. .. _extra-facets-example-1: .. code-block:: yaml - :caption: Extra facet example file `native6-era5.yml` + :caption: Extra facet example file `native6-era5-example.yml` ERA5: Amon: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index b7708fd95f..d93f114f21 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -107,18 +107,27 @@ The following native reanalysis/observational datasets are supported under the To use these datasets, put the files containing the data in the directory that you have :ref:`configured ` for the ``rootpath`` of the ``native6`` project, in a subdirectory called -``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}``. +``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}`` (assuming you are +using the ``default`` DRS for ``native6``). Replace the items in curly braces by the values used in the variable/dataset definition in the :ref:`recipe `. -Below is a list of native reanalysis/observational datasets currently -supported. -.. _read_native_era5: +.. _read_native_era5_nc: -ERA5 -^^^^ +ERA5 (in netCDF format downloaded from the CDS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERA5 data can be downloaded from the Copernicus Climate Data Store (CDS) using +the convenient tool `era5cli `__. +For example for monthly data, place the files in the +``/Tier3/ERA5/version/mon/pr`` subdirectory of your ``rootpath`` that you have +configured for the ``native6`` project (assuming you are using the ``default`` +DRS for ``native6``). -- Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``) +- Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, + ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, + ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, + ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``). - Tier: 3 .. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page @@ -131,6 +140,85 @@ ERA5 of both liquid and solid phases to vapor (from underlying surface and vegetation)." Therefore, the ERA5 (and ERA5-Land) CMORizer switches the signs of ``evspsbl`` and ``evspsblpot`` to be compatible with the CMOR standard used e.g. by the CMIP models. +.. _read_native_era5_grib: + +ERA5 (in GRIB format available on DKRZ's Levante or downloaded from the CDS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERA5 data in monthly, daily, and hourly resolution is `available on Levante +`__ +in its native GRIB format. + +.. note:: + ERA5 data in its native GRIB format can also be downloaded from the + `Copernicus Climate Data Store (CDS) + `__. + For example, hourly data on pressure levels is available `here + `__. + Reading self-downloaded ERA5 data in GRIB format is experimental and likely + requires additional setup from the user like setting up the proper directory + structure for the input files and/or creating a custom :ref:`DRS + `. + +To read these data with ESMValCore, use the :ref:`rootpath +` ``/pool/data/ERA5`` with :ref:`DRS +` ``DKRZ-ERA5-GRIB`` in your configuration, for example: + +.. code-block:: yaml + + rootpath: + ... + native6: + /pool/data/ERA5: DKRZ-ERA5-GRIB + ... + +The `naming conventions +`__ +for input directories and files for native ERA5 data in GRIB format on Levante +are + +* input directories: ``{family}/{level}/{type}/{tres}/{grib_id}`` +* input files: ``{family}{level}{typeid}_{tres}_*_{grib_id}.grb`` + +All of these facets have reasonable defaults preconfigured in the corresponding +:ref:`extra facets` file, which is available here: +:download:`native6-era5.yml +`. +If necessary, these facets can be overwritten in the recipe. + +Thus, example dataset entries could look like this: + +.. code-block:: yaml + + datasets: + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: cl, mip: Amon, tres: 1H, frequency: 1hr} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: ta, mip: Amon, type: fc, typeid: '12'} + +The native ERA5 output in GRIB format is stored on a `reduced Gaussian grid +`__. +By default, these data are regridded to a regular 0.25°x0.25° grid as +`recommended by the ECMWF +`__ +using bilinear interpolation. + +To disable this, you can use the facet ``automatic_regrid: false`` in the +recipe: + +.. code-block:: yaml + + datasets: + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon, automatic_regrid: false} + +- Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, + ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, + ``snd``, ``snowmxrat27``, ``ta``, ``tas``, ``tdps``, ``toz``, ``ts``, ``ua``, + ``uas``, ``va``, ``vas``, ``wap``, ``zg``. + .. _read_native_mswep: MSWEP @@ -140,7 +228,10 @@ MSWEP - Supported frequencies: ``mon``, ``day``, ``3hr``. - Tier: 3 -For example for monthly data, place the files in the ``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``native6`` project location. +For example for monthly data, place the files in the +``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``rootpath`` that you have +configured for the ``native6`` project (assuming you are using the ``default`` +DRS for ``native6``). .. note:: For monthly data (``V220``), the data must be postfixed with the date, i.e. rename ``global_monthly_050deg.nc`` to ``global_monthly_050deg_197901-201710.nc`` @@ -642,6 +733,8 @@ first discuss the ``drs`` parameter: as we've seen in the previous section, the DRS as a standard is used for both file naming conventions and for directory structures. +.. _config_option_drs: + Explaining ``drs: CMIP5:`` or ``drs: CMIP6:`` --------------------------------------------- Whereas ESMValCore will by default use the CMOR standard for file naming (please diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 25ad81f5ba..89b5822c27 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -4,6 +4,7 @@ import logging import os from functools import total_ordering +from pathlib import Path from netCDF4 import Dataset from PIL import Image @@ -209,9 +210,10 @@ def _initialize_entity(self): """Initialize the entity representing the file.""" if self.attributes is None: self.attributes = {} - with Dataset(self.filename, "r") as dataset: - for attr in dataset.ncattrs(): - self.attributes[attr] = dataset.getncattr(attr) + if "nc" in Path(self.filename).suffix: + with Dataset(self.filename, "r") as dataset: + for attr in dataset.ncattrs(): + self.attributes[attr] = dataset.getncattr(attr) attributes = { "attribute:" + str(k).replace(" ", "_"): str(v) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 8d4809ffa0..9c5aa74553 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -37,6 +37,7 @@ PreprocessorFile, ) from esmvalcore.preprocessor._area import _update_shapefile_path +from esmvalcore.preprocessor._io import GRIB_FORMATS from esmvalcore.preprocessor._multimodel import _get_stat_identifier from esmvalcore.preprocessor._regrid import ( _spec_to_latlonvals, @@ -230,6 +231,34 @@ def _get_default_settings(dataset): return settings +def _add_dataset_specific_settings(dataset: Dataset, settings: dict) -> None: + """Add dataset-specific settings.""" + project = dataset.facets["project"] + dataset_name = dataset.facets["dataset"] + file_suffixes = [Path(file.name).suffix for file in dataset.files] + + # Automatic regridding for native ERA5 data in GRIB format if regridding + # step is not already present (can be disabled with facet + # automatic_regrid=False) + if all( + [ + project == "native6", + dataset_name == "ERA5", + any(grib_format in file_suffixes for grib_format in GRIB_FORMATS), + "regrid" not in settings, + dataset.facets.get("automatic_regrid", True), + ] + ): + # Settings recommended by ECMWF + # (https://confluence.ecmwf.int/display/CKB/ERA5%3A+What+is+the+spatial+reference#heading-Interpolation) + settings["regrid"] = {"target_grid": "0.25x0.25", "scheme": "linear"} + logger.debug( + "Automatically regrid native6 ERA5 data in GRIB format with the " + "settings %s", + settings["regrid"], + ) + + def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { @@ -546,6 +575,7 @@ def _get_preprocessor_products( _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(dataset.facets, settings) _update_preproc_functions(settings, dataset, datasets, missing_vars) + _add_dataset_specific_settings(dataset, settings) check.preprocessor_supplementaries(dataset, settings) input_datasets = _get_input_datasets(dataset) missing = _check_input_files(input_datasets) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 4d3e297e3a..9a229b2dc4 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -845,6 +845,8 @@ def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: """Fix time bounds.""" times = {"time", "time1", "time2", "time3"} key = times.intersection(self.vardef.coordinates) + if not key: # cube has time, but CMOR variable does not + return cmor = self.vardef.coordinates[" ".join(key)] if cmor.must_have_bounds == "yes" and not cube_coord.has_bounds(): cube_coord.bounds = get_time_bounds(cube_coord, self.frequency) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 85b570c57d..2214238557 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -5,12 +5,16 @@ import iris import numpy as np +from iris.util import reverse -from esmvalcore.iris_helpers import date2num, safe_convert_units - -from ...table import CMOR_TABLES -from ..fix import Fix -from ..shared import add_scalar_height_coord +from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord +from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.iris_helpers import ( + date2num, + has_unstructured_grid, + safe_convert_units, +) logger = logging.getLogger(__name__) @@ -24,7 +28,11 @@ def get_frequency(cube): time.convert_units("days since 1850-1-1 00:00:00.0") if len(time.points) == 1: - if cube.long_name != "Geopotential": + acceptable_long_names = ( + "Geopotential", + "Percentage of the Grid Cell Occupied by Land (Including Lakes)", + ) + if cube.long_name not in acceptable_long_names: raise ValueError( "Unable to infer frequency of cube " f"with length 1 time dimension: {cube}" @@ -32,9 +40,11 @@ def get_frequency(cube): return "fx" interval = time.points[1] - time.points[0] + if interval - 1 / 24 < 1e-4: return "hourly" - + if interval - 1.0 < 1e-4: + return "daily" return "monthly" @@ -52,6 +62,11 @@ def fix_accumulated_units(cube): cube.units = cube.units * "d-1" elif get_frequency(cube) == "hourly": cube.units = cube.units * "h-1" + elif get_frequency(cube) == "daily": + raise NotImplementedError( + f"Fixing of accumulated units of cube " + f"{cube.summary(shorten=True)} is not implemented for daily data" + ) return cube @@ -76,6 +91,27 @@ def divide_by_gravity(cube): return cube +class Albsn(Fix): + """Fixes for albsn.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Invalid input cube units (ignored on load) were '0-1' + cube.units = "1" + return cubes + + +class Cli(Fix): + """Fixes for cli.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg kg-1" + return cubes + + class Clt(Fix): """Fixes for clt.""" @@ -89,6 +125,16 @@ def fix_metadata(self, cubes): return cubes +class Clw(Fix): + """Fixes for clw.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg kg-1" + return cubes + + class Cl(Fix): """Fixes for cl.""" @@ -136,6 +182,16 @@ def fix_metadata(self, cubes): return cubes +class Hus(Fix): + """Fixes for hus.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg kg-1" + return cubes + + class Mrro(Fix): """Fixes for mrro.""" @@ -149,6 +205,20 @@ def fix_metadata(self, cubes): return cubes +class O3(Fix): + """Fixes for o3.""" + + def fix_metadata(self, cubes): + """Convert mass mixing ratios to mole fractions.""" + for cube in cubes: + # Original units are kg kg-1. Convert these to molar mixing ratios, + # which is almost identical to mole fraction for small amounts of + # substances (which we have here) + cube.data = cube.core_data() * 28.9644 / 47.9982 + cube.units = "mol mol-1" + return cubes + + class Orog(Fix): """Fixes for orography.""" @@ -194,6 +264,26 @@ def fix_metadata(self, cubes): return cubes +class Prw(Fix): + """Fixes for prw.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg m-2" + return cubes + + +class Ps(Fix): + """Fixes for ps.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "Pa" + return cubes + + class Ptype(Fix): """Fixes for ptype.""" @@ -205,6 +295,16 @@ def fix_metadata(self, cubes): return cubes +class Rainmxrat27(Fix): + """Fixes for rainmxrat27.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg kg-1" + return cubes + + class Rlds(Fix): """Fixes for Rlds.""" @@ -321,6 +421,27 @@ def fix_metadata(self, cubes): return cubes +class Sftlf(Fix): + """Fixes for sftlf.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Invalid input cube units (ignored on load) were '0-1' + cube.units = "1" + return cubes + + +class Snowmxrat27(Fix): + """Fixes for snowmxrat27.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = "kg kg-1" + return cubes + + class Tasmax(Fix): """Fixes for tasmax.""" @@ -341,6 +462,22 @@ def fix_metadata(self, cubes): return cubes +class Toz(Fix): + """Fixes for toz.""" + + def fix_metadata(self, cubes): + """Convert 'kg m-2' to 'm'.""" + for cube in cubes: + # Original units are kg m-2. Convert these to m here. + # 1 DU = 0.4462 mmol m-2 = 21.415 mg m-2 = 2.1415e-5 kg m-2 + # (assuming O3 molar mass of 48 g mol-1) + # Since 1 mm of pure O3 layer is defined as 100 DU + # --> 1m ~ 2.1415 kg m-2 + cube.data = cube.core_data() / 2.1415 + cube.units = "m" + return cubes + + class Zg(Fix): """Fixes for Geopotential.""" @@ -356,21 +493,13 @@ class AllVars(Fix): def _fix_coordinates(self, cube): """Fix coordinates.""" - # Fix coordinate increasing direction - slices = [] - for coord in cube.coords(): - if coord.var_name in ("latitude", "pressure_level"): - slices.append(slice(None, None, -1)) - else: - slices.append(slice(None)) - cube = cube[tuple(slices)] - # Add scalar height coordinates if "height2m" in self.vardef.dimensions: add_scalar_height_coord(cube, 2.0) if "height10m" in self.vardef.dimensions: add_scalar_height_coord(cube, 10.0) + # Fix coord metadata for coord_def in self.vardef.coordinates.values(): axis = coord_def.axis # ERA5 uses regular pressure level coordinate. In case the cmor @@ -383,7 +512,7 @@ def _fix_coordinates(self, cube): coord = cube.coord(axis=axis) if axis == "T": coord.convert_units("days since 1850-1-1 00:00:00.0") - if axis == "Z": + if axis in ("X", "Y", "Z"): coord.convert_units(coord_def.units) coord.standard_name = coord_def.standard_name coord.var_name = coord_def.out_name @@ -394,10 +523,25 @@ def _fix_coordinates(self, cube): and len(coord.core_points()) > 1 and coord_def.must_have_bounds == "yes" ): - coord.guess_bounds() + # Do not guess bounds for lat and lon on unstructured grids + if not ( + coord.name() in ("latitude", "longitude") + and has_unstructured_grid(cube) + ): + coord.guess_bounds() self._fix_monthly_time_coord(cube) + # Fix coordinate increasing direction + if cube.coords("latitude") and not has_unstructured_grid(cube): + lat = cube.coord("latitude") + if lat.points[0] > lat.points[-1]: + cube = reverse(cube, "latitude") + if cube.coords("air_pressure"): + plev = cube.coord("air_pressure") + if plev.points[0] < plev.points[-1]: + cube = reverse(cube, "air_pressure") + return cube @staticmethod @@ -426,16 +570,18 @@ def fix_metadata(self, cubes): if self.vardef.standard_name: cube.standard_name = self.vardef.standard_name cube.long_name = self.vardef.long_name - cube = self._fix_coordinates(cube) cube = safe_convert_units(cube, self.vardef.units) - cube.data = cube.core_data().astype("float32") year = datetime.datetime.now().year cube.attributes["comment"] = ( "Contains modified Copernicus Climate Change " f"Service Information {year}" ) + if "GRIB_PARAM" in cube.attributes: + cube.attributes["GRIB_PARAM"] = str( + cube.attributes["GRIB_PARAM"] + ) fixed_cubes.append(cube) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index c81324142a..faa009ec8f 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -99,8 +99,10 @@ native6: cmor_strict: false input_dir: default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' + DKRZ-ERA5-GRIB: '{family}/{level}/{type}/{tres}/{grib_id}' input_file: default: '*.nc' + DKRZ-ERA5-GRIB: '{family}{level}{typeid}_{tres}_*_{grib_id}.grb' output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/esmvalcore/config/configurations/defaults/config-user.yml b/esmvalcore/config/configurations/defaults/config-user.yml index 39cffb67fb..a666875542 100644 --- a/esmvalcore/config/configurations/defaults/config-user.yml +++ b/esmvalcore/config/configurations/defaults/config-user.yml @@ -196,7 +196,9 @@ drs: # /work/bd0854/DATA/ESMValTool2/OBS: default # /work/bd0854/DATA/ESMValTool2/download: ESGF # ana4mips: /work/bd0854/DATA/ESMValTool2/OBS -# native6: /work/bd0854/DATA/ESMValTool2/RAWOBS +# native6: +# /work/bd0854/DATA/ESMValTool2/RAWOBS: default +# /pool/data/ERA5: DKRZ-ERA5-GRIB # RAWOBS: /work/bd0854/DATA/ESMValTool2/RAWOBS #drs: # ana4mips: default diff --git a/esmvalcore/config/extra_facets/native6-era5.yml b/esmvalcore/config/extra_facets/native6-era5.yml new file mode 100644 index 0000000000..4ab1915da9 --- /dev/null +++ b/esmvalcore/config/extra_facets/native6-era5.yml @@ -0,0 +1,196 @@ +# Extra facets for native6 ERA5 data in GRIB format +# +# See +# https://docs.dkrz.de/doc/dataservices/finding_and_accessing_data/era_data/index.html#file-and-directory-names +# for details on these facets. + +# Notes: +# - All facets can also be specified in the recipes. The values given here are +# only defaults. + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). +--- + +ERA5: + + # Settings for all variables of all MIPs + '*': + '*': + automatic_regrid: true + family: E5 + type: an + typeid: '00' + version: v1 + + # Variable-specific settings + albsn: + level: sf + grib_id: '032' + cl: + level: pl + grib_id: '248' + cli: + level: pl + grib_id: '247' + clt: + level: sf + grib_id: '164' + clw: + level: pl + grib_id: '246' + hur: + level: pl + grib_id: '157' + hus: + level: pl + grib_id: '133' + o3: + level: pl + grib_id: '203' + prw: + level: sf + grib_id: '137' + ps: + level: sf + grib_id: '134' + psl: + level: sf + grib_id: '151' + rainmxrat27: + level: pl + grib_id: '075' + sftlf: + level: sf + grib_id: '172' + siconc: + level: sf + grib_id: '031' + siconca: + level: sf + grib_id: '031' + snd: + level: sf + grib_id: '141' + snowmxrat27: + level: pl + grib_id: '076' + ta: + level: pl + grib_id: '130' + tas: + level: sf + grib_id: '167' + tdps: + level: sf + grib_id: '168' + tos: + level: sf + grib_id: '034' + toz: + level: sf + grib_id: '206' + ts: + level: sf + grib_id: '235' + ua: + level: pl + grib_id: '131' + uas: + level: sf + grib_id: '165' + va: + level: pl + grib_id: '132' + vas: + level: sf + grib_id: '166' + wap: + level: pl + grib_id: '135' + zg: + level: pl + grib_id: '129' + + # MIP-specific settings + AERday: + '*': + tres: 1D + AERhr: + '*': + tres: 1H + AERmon: + '*': + tres: 1M + AERmonZ: + '*': + tres: 1M + Amon: + '*': + tres: 1M + CFday: + '*': + tres: 1D + CFmon: + '*': + tres: 1M + day: + '*': + tres: 1D + E1hr: + '*': + tres: 1H + E1hrClimMon: + '*': + tres: 1H + Eday: + '*': + tres: 1D + EdayZ: + '*': + tres: 1D + Efx: + '*': + tres: IV + Emon: + '*': + tres: 1M + EmonZ: + '*': + tres: 1M + fx: + '*': + tres: IV + IfxAnt: + '*': + tres: IV + IfxGre: + '*': + tres: IV + ImonAnt: + '*': + tres: 1M + ImonGre: + '*': + tres: 1M + LImon: + '*': + tres: 1M + Lmon: + '*': + tres: 1M + Oday: + '*': + tres: 1D + Ofx: + '*': + tres: IV + Omon: + '*': + tres: 1M + SIday: + '*': + tres: 1D + SImon: + '*': + tres: 1M diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index eccac411f2..0851e1d37e 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -39,6 +39,7 @@ "reference_dataset", "alternative_dataset", } +GRIB_FORMATS = (".grib2", ".grib", ".grb2", ".grb", ".gb2", ".gb") iris.FUTURE.save_split_attrs = True @@ -142,7 +143,13 @@ def load( # warnings.filterwarnings # (see https://github.com/SciTools/cf-units/issues/240) with suppress_errors(): - raw_cubes = iris.load_raw(file, callback=_load_callback) + # GRIB files need to be loaded with iris.load, otherwise we will + # get separate (lat, lon) slices for each time step, pressure + # level, etc. + if file.suffix in GRIB_FORMATS: + raw_cubes = iris.load(file, callback=_load_callback) + else: + raw_cubes = iris.load_raw(file, callback=_load_callback) logger.debug("Done with loading %s", file) if not raw_cubes: diff --git a/pyproject.toml b/pyproject.toml index 61de91ae25..4ce0f4c303 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,7 +168,7 @@ enable_error_code = [ # Configure linters [tool.codespell] -skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" +skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml,tests/sample_data/iris-sample-data/LICENSE" ignore-words-list = "vas,hist,oce" [tool.ruff] diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 60460138a9..b65acabbff 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -2,16 +2,19 @@ import datetime -import iris +import dask.array as da import numpy as np import pytest from cf_units import Unit +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor._fixes.native6.era5 import ( AllVars, Evspsbl, Zg, + fix_accumulated_units, get_frequency, ) from esmvalcore.cmor.fix import fix_metadata @@ -40,12 +43,12 @@ def test_get_zg_fix(): def test_get_frequency_hourly(): """Test cubes with hourly frequency.""" - time = iris.coords.DimCoord( + time = DimCoord( [0, 1, 2], standard_name="time", units=Unit("hours since 1900-01-01"), ) - cube = iris.cube.Cube( + cube = Cube( [1, 6, 3], var_name="random_var", dim_coords_and_dims=[(time, 0)], @@ -55,14 +58,31 @@ def test_get_frequency_hourly(): assert get_frequency(cube) == "hourly" +def test_get_frequency_daily(): + """Test cubes with daily frequency.""" + time = DimCoord( + [0, 1, 2], + standard_name="time", + units=Unit("days since 1900-01-01"), + ) + cube = Cube( + [1, 6, 3], + var_name="random_var", + dim_coords_and_dims=[(time, 0)], + ) + assert get_frequency(cube) == "daily" + cube.coord("time").convert_units("hours since 1850-1-1 00:00:00.0") + assert get_frequency(cube) == "daily" + + def test_get_frequency_monthly(): """Test cubes with monthly frequency.""" - time = iris.coords.DimCoord( + time = DimCoord( [0, 31, 59], standard_name="time", units=Unit("hours since 1900-01-01"), ) - cube = iris.cube.Cube( + cube = Cube( [1, 6, 3], var_name="random_var", dim_coords_and_dims=[(time, 0)], @@ -74,27 +94,50 @@ def test_get_frequency_monthly(): def test_get_frequency_fx(): """Test cubes with time invariant frequency.""" - cube = iris.cube.Cube(1.0, long_name="Cube without time coordinate") + cube = Cube(1.0, long_name="Cube without time coordinate") assert get_frequency(cube) == "fx" - time = iris.coords.DimCoord( + + time = DimCoord( 0, standard_name="time", units=Unit("hours since 1900-01-01"), ) - cube = iris.cube.Cube( + cube = Cube( [1], var_name="cube_with_length_1_time_coord", long_name="Geopotential", dim_coords_and_dims=[(time, 0)], ) assert get_frequency(cube) == "fx" + + cube.long_name = ( + "Percentage of the Grid Cell Occupied by Land (Including Lakes)" + ) + assert get_frequency(cube) == "fx" + cube.long_name = "Not geopotential" with pytest.raises(ValueError): get_frequency(cube) +def test_fix_accumulated_units_fail(): + """Test `fix_accumulated_units`.""" + time = DimCoord( + [0, 1, 2], + standard_name="time", + units=Unit("days since 1900-01-01"), + ) + cube = Cube( + [1, 6, 3], + var_name="random_var", + dim_coords_and_dims=[(time, 0)], + ) + with pytest.raises(NotImplementedError): + fix_accumulated_units(cube) + + def _era5_latitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([90.0, 0.0, -90.0]), standard_name="latitude", long_name="latitude", @@ -104,7 +147,7 @@ def _era5_latitude(): def _era5_longitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([0, 180, 359.75]), standard_name="longitude", long_name="longitude", @@ -117,11 +160,15 @@ def _era5_longitude(): def _era5_time(frequency): if frequency == "invariant": timestamps = [788928] # hours since 1900 at 1 january 1990 + elif frequency == "daily": + timestamps = [788940, 788964, 788988] elif frequency == "hourly": timestamps = [788928, 788929, 788930] elif frequency == "monthly": timestamps = [788928, 789672, 790344] - return iris.coords.DimCoord( + else: + raise NotImplementedError(f"Invalid frequency {frequency}") + return DimCoord( np.array(timestamps, dtype="int32"), standard_name="time", long_name="time", @@ -137,7 +184,7 @@ def _era5_plev(): 1000, ] ) - return iris.coords.DimCoord( + return DimCoord( values, long_name="pressure", units=Unit("millibars"), @@ -153,7 +200,7 @@ def _era5_data(frequency): def _cmor_latitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([-90.0, 0.0, 90.0]), standard_name="latitude", long_name="Latitude", @@ -164,7 +211,7 @@ def _cmor_latitude(): def _cmor_longitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([0, 180, 359.75]), standard_name="longitude", long_name="Longitude", @@ -184,14 +231,22 @@ def _cmor_time(mip, bounds=None, shifted=False): timestamps -= 1 / 48 if bounds is not None: bounds = [[t - 1 / 48, t + 1 / 48] for t in timestamps] - elif mip == "Amon": + elif mip == "Eday": + timestamps = np.array([51134.5, 51135.5, 51136.5]) + if bounds is not None: + bounds = np.array( + [[51134.0, 51135.0], [51135.0, 51136.0], [51136.0, 51137.0]] + ) + elif "mon" in mip: timestamps = np.array([51149.5, 51179.0, 51208.5]) if bounds is not None: bounds = np.array( [[51134.0, 51165.0], [51165.0, 51193.0], [51193.0, 51224.0]] ) + else: + raise NotImplementedError() - return iris.coords.DimCoord( + return DimCoord( np.array(timestamps, dtype=float), standard_name="time", long_name="time", @@ -202,7 +257,7 @@ def _cmor_time(mip, bounds=None, shifted=False): def _cmor_aux_height(value): - return iris.coords.AuxCoord( + return AuxCoord( value, long_name="height", standard_name="height", @@ -219,7 +274,7 @@ def _cmor_plev(): 100.0, ] ) - return iris.coords.DimCoord( + return DimCoord( values, long_name="pressure", standard_name="air_pressure", @@ -235,10 +290,97 @@ def _cmor_data(mip): return np.arange(27).reshape(3, 3, 3)[:, ::-1, :] +def era5_2d(frequency): + if frequency == "monthly": + time = DimCoord( + [-31, 0, 31], standard_name="time", units="days since 1850-01-01" + ) + else: + time = _era5_time(frequency) + cube = Cube( + _era5_data(frequency), + long_name=None, + var_name=None, + units="unknown", + dim_coords_and_dims=[ + (time, 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def era5_3d(frequency): + cube = Cube( + np.ones((3, 2, 3, 3)), + long_name=None, + var_name=None, + units="unknown", + dim_coords_and_dims=[ + (_era5_time(frequency), 0), + (_era5_plev(), 1), + (_era5_latitude(), 2), + (_era5_longitude(), 3), + ], + ) + return CubeList([cube]) + + +def cmor_2d(mip, short_name): + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + if "mon" in mip: + time = DimCoord( + [-15.5, 15.5, 45.0], + bounds=[[-31.0, 0.0], [0.0, 31.0], [31.0, 59.0]], + standard_name="time", + long_name="time", + var_name="time", + units="days since 1850-01-01", + ) + else: + time = _cmor_time(mip, bounds=True) + cube = Cube( + _cmor_data(mip).astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT}, + ) + return CubeList([cube]) + + +def cmor_3d(mip, short_name): + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + cube = Cube( + np.ones((3, 2, 3, 3)), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (_cmor_time(mip, bounds=True), 0), + (_cmor_plev(), 1), + (_cmor_latitude(), 2), + (_cmor_longitude(), 3), + ], + attributes={"comment": COMMENT}, + ) + return CubeList([cube]) + + def cl_era5_monthly(): time = _era5_time("monthly") data = np.ones((3, 2, 3, 3)) - cube = iris.cube.Cube( + cube = Cube( data, long_name="Percentage Cloud Cover", var_name="cl", @@ -250,7 +392,7 @@ def cl_era5_monthly(): (_era5_longitude(), 3), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def cl_cmor_amon(): @@ -259,7 +401,7 @@ def cl_cmor_amon(): time = _cmor_time("Amon", bounds=True) data = np.ones((3, 2, 3, 3)) data = data * 100.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -273,12 +415,12 @@ def cl_cmor_amon(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def clt_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="cloud cover fraction", var_name="cloud_cover", @@ -289,7 +431,7 @@ def clt_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def clt_cmor_e1hr(): @@ -297,7 +439,7 @@ def clt_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "clt") time = _cmor_time("E1hr", bounds=True) data = _cmor_data("E1hr") * 100 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -310,12 +452,12 @@ def clt_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsbl_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly") * -1.0, long_name="total evapotranspiration", var_name="e", @@ -326,7 +468,7 @@ def evspsbl_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsbl_cmor_e1hr(): @@ -334,7 +476,7 @@ def evspsbl_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "evspsbl") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") * 1000 / 3600.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -347,12 +489,12 @@ def evspsbl_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsblpot_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly") * -1.0, long_name="potential evapotranspiration", var_name="epot", @@ -363,7 +505,7 @@ def evspsblpot_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsblpot_cmor_e1hr(): @@ -371,7 +513,7 @@ def evspsblpot_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "evspsblpot") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") * 1000 / 3600.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -384,12 +526,12 @@ def evspsblpot_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def mrro_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="runoff", var_name="runoff", @@ -400,7 +542,7 @@ def mrro_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def mrro_cmor_e1hr(): @@ -408,7 +550,7 @@ def mrro_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "mrro") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") * 1000 / 3600.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -421,12 +563,20 @@ def mrro_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) + + +def o3_era5_monthly(): + cube = era5_3d("monthly")[0] + cube = cube[:, ::-1, ::-1, :] # test if correct order of plev and lat stay + cube.data = cube.data.astype("float32") + cube.data *= 47.9982 / 28.9644 + return CubeList([cube]) def orog_era5_hourly(): time = _era5_time("invariant") - cube = iris.cube.Cube( + cube = Cube( _era5_data("invariant"), long_name="geopotential height", var_name="zg", @@ -437,14 +587,14 @@ def orog_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def orog_cmor_fx(): cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable("fx", "orog") data = _cmor_data("fx") / 9.80665 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -453,12 +603,12 @@ def orog_cmor_fx(): dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_era5_monthly(): time = _era5_time("monthly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("monthly"), long_name="total_precipitation", var_name="tp", @@ -469,7 +619,7 @@ def pr_era5_monthly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_cmor_amon(): @@ -477,7 +627,7 @@ def pr_cmor_amon(): vardef = cmor_table.get_variable("Amon", "pr") time = _cmor_time("Amon", bounds=True) data = _cmor_data("Amon") * 1000.0 / 3600.0 / 24.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -490,12 +640,12 @@ def pr_cmor_amon(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="total_precipitation", var_name="tp", @@ -506,7 +656,7 @@ def pr_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_cmor_e1hr(): @@ -514,7 +664,7 @@ def pr_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "pr") time = _cmor_time("E1hr", bounds=True, shifted=True) data = _cmor_data("E1hr") * 1000.0 / 3600.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -527,12 +677,12 @@ def pr_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def prsn_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="snow", var_name="snow", @@ -543,7 +693,7 @@ def prsn_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def prsn_cmor_e1hr(): @@ -551,7 +701,7 @@ def prsn_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "prsn") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") * 1000 / 3600.0 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -564,12 +714,12 @@ def prsn_cmor_e1hr(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def ptype_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="snow", var_name="snow", @@ -580,7 +730,7 @@ def ptype_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def ptype_cmor_e1hr(): @@ -588,7 +738,7 @@ def ptype_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "ptype") time = _cmor_time("E1hr", shifted=False, bounds=True) data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -602,12 +752,12 @@ def ptype_cmor_e1hr(): ) cube.coord("latitude").long_name = "latitude" cube.coord("longitude").long_name = "longitude" - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rlds_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="surface thermal radiation downwards", var_name="ssrd", @@ -618,7 +768,7 @@ def rlds_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rlds_cmor_e1hr(): @@ -626,7 +776,87 @@ def rlds_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "rlds") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") / 3600 - cube = iris.cube.Cube( + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + return CubeList([cube]) + + +def rlns_era5_hourly(): + freq = "hourly" + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units="J m**-2", + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rlns_cmor_e1hr(): + mip = "E1hr" + short_name = "rlns" + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + cube.coord("latitude").long_name = "latitude" # from custom table + cube.coord("longitude").long_name = "longitude" # from custom table + return CubeList([cube]) + + +def rlus_era5_hourly(): + freq = "hourly" + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units="J m**-2", + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rlus_cmor_e1hr(): + mip = "E1hr" + short_name = "rlus" + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -637,17 +867,14 @@ def rlds_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={ - "comment": COMMENT, - "positive": "down", - }, + attributes={"comment": COMMENT, "positive": "up"}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rls_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="runoff", var_name="runoff", @@ -658,7 +885,7 @@ def rls_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rls_cmor_e1hr(): @@ -666,7 +893,7 @@ def rls_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "rls") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -677,17 +904,14 @@ def rls_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={ - "comment": COMMENT, - "positive": "down", - }, + attributes={"comment": COMMENT, "positive": "down"}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsds_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="solar_radiation_downwards", var_name="rlwd", @@ -698,7 +922,7 @@ def rsds_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsds_cmor_e1hr(): @@ -706,7 +930,87 @@ def rsds_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "rsds") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") / 3600 - cube = iris.cube.Cube( + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + return CubeList([cube]) + + +def rsns_era5_hourly(): + freq = "hourly" + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units="J m**-2", + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rsns_cmor_e1hr(): + mip = "E1hr" + short_name = "rsns" + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + cube.coord("latitude").long_name = "latitude" # from custom table + cube.coord("longitude").long_name = "longitude" # from custom table + return CubeList([cube]) + + +def rsus_era5_hourly(): + freq = "hourly" + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units="J m**-2", + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rsus_cmor_e1hr(): + mip = "E1hr" + short_name = "rsus" + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -717,17 +1021,14 @@ def rsds_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={ - "comment": COMMENT, - "positive": "down", - }, + attributes={"comment": COMMENT, "positive": "up"}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsdt_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="thermal_radiation_downwards", var_name="strd", @@ -738,7 +1039,7 @@ def rsdt_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsdt_cmor_e1hr(): @@ -746,7 +1047,7 @@ def rsdt_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "rsdt") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") / 3600 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -757,17 +1058,14 @@ def rsdt_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={ - "comment": COMMENT, - "positive": "down", - }, + attributes={"comment": COMMENT, "positive": "down"}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rss_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="net_solar_radiation", var_name="ssr", @@ -778,7 +1076,7 @@ def rss_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rss_cmor_e1hr(): @@ -786,7 +1084,7 @@ def rss_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "rss") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") / 3600 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -797,17 +1095,43 @@ def rss_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={ - "comment": COMMENT, - "positive": "down", - }, + attributes={"comment": COMMENT, "positive": "down"}, + ) + return CubeList([cube]) + + +def sftlf_era5(): + cube = Cube( + np.ones((3, 3)), + long_name=None, + var_name=None, + units="unknown", + dim_coords_and_dims=[ + (_era5_latitude(), 0), + (_era5_longitude(), 1), + ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) + + +def sftlf_cmor_fx(): + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("fx", "sftlf") + cube = Cube( + np.ones((3, 3)).astype("float32") * 100.0, + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], + attributes={"comment": COMMENT}, + ) + return CubeList([cube]) def tas_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="2m_temperature", var_name="t2m", @@ -818,7 +1142,7 @@ def tas_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_cmor_e1hr(): @@ -826,7 +1150,7 @@ def tas_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "tas") time = _cmor_time("E1hr") data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -840,12 +1164,12 @@ def tas_cmor_e1hr(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_era5_monthly(): time = _era5_time("monthly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("monthly"), long_name="2m_temperature", var_name="t2m", @@ -856,7 +1180,7 @@ def tas_era5_monthly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_cmor_amon(): @@ -864,7 +1188,7 @@ def tas_cmor_amon(): vardef = cmor_table.get_variable("Amon", "tas") time = _cmor_time("Amon", bounds=True) data = _cmor_data("Amon") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -878,13 +1202,20 @@ def tas_cmor_amon(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) + + +def toz_era5_monthly(): + cube = era5_2d("monthly")[0] + cube.data = cube.data.astype("float32") + cube.data *= 2.1415 + return CubeList([cube]) def zg_era5_monthly(): time = _era5_time("monthly") data = np.ones((3, 2, 3, 3)) - cube = iris.cube.Cube( + cube = Cube( data, long_name="geopotential height", var_name="zg", @@ -896,7 +1227,7 @@ def zg_era5_monthly(): (_era5_longitude(), 3), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def zg_cmor_amon(): @@ -905,7 +1236,7 @@ def zg_cmor_amon(): time = _cmor_time("Amon", bounds=True) data = np.ones((3, 2, 3, 3)) data = data / 9.80665 - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -919,12 +1250,12 @@ def zg_cmor_amon(): ], attributes={"comment": COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmax_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="maximum 2m temperature", var_name="mx2t", @@ -935,7 +1266,7 @@ def tasmax_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmax_cmor_e1hr(): @@ -943,7 +1274,7 @@ def tasmax_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "tasmax") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -957,12 +1288,12 @@ def tasmax_cmor_e1hr(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmin_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="minimum 2m temperature", var_name="mn2t", @@ -973,7 +1304,7 @@ def tasmin_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmin_cmor_e1hr(): @@ -981,7 +1312,7 @@ def tasmin_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "tasmin") time = _cmor_time("E1hr", shifted=True, bounds=True) data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -995,12 +1326,12 @@ def tasmin_cmor_e1hr(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def uas_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="10m_u_component_of_wind", var_name="u10", @@ -1011,7 +1342,7 @@ def uas_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def uas_cmor_e1hr(): @@ -1019,7 +1350,7 @@ def uas_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "uas") time = _cmor_time("E1hr") data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -1033,12 +1364,12 @@ def uas_cmor_e1hr(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(10.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def vas_era5_hourly(): time = _era5_time("hourly") - cube = iris.cube.Cube( + cube = Cube( _era5_data("hourly"), long_name="10m_v_component_of_wind", var_name="v10", @@ -1049,7 +1380,7 @@ def vas_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def vas_cmor_e1hr(): @@ -1057,7 +1388,7 @@ def vas_cmor_e1hr(): vardef = cmor_table.get_variable("E1hr", "vas") time = _cmor_time("E1hr") data = _cmor_data("E1hr") - cube = iris.cube.Cube( + cube = Cube( data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, @@ -1071,14 +1402,17 @@ def vas_cmor_e1hr(): attributes={"comment": COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(10.0)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) VARIABLES = [ pytest.param(a, b, c, d, id=c + "_" + d) for (a, b, c, d) in [ + (era5_2d("daily"), cmor_2d("Eday", "albsn"), "albsn", "Eday"), (cl_era5_monthly(), cl_cmor_amon(), "cl", "Amon"), + (era5_3d("monthly"), cmor_3d("Amon", "cli"), "cli", "Amon"), (clt_era5_hourly(), clt_cmor_e1hr(), "clt", "E1hr"), + (era5_3d("monthly"), cmor_3d("Amon", "clw"), "clw", "Amon"), (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), "evspsbl", "E1hr"), ( evspsblpot_era5_hourly(), @@ -1086,21 +1420,43 @@ def vas_cmor_e1hr(): "evspsblpot", "E1hr", ), + (era5_3d("monthly"), cmor_3d("Amon", "hus"), "hus", "Amon"), (mrro_era5_hourly(), mrro_cmor_e1hr(), "mrro", "E1hr"), + (o3_era5_monthly(), cmor_3d("Amon", "o3"), "o3", "Amon"), (orog_era5_hourly(), orog_cmor_fx(), "orog", "fx"), (pr_era5_monthly(), pr_cmor_amon(), "pr", "Amon"), (pr_era5_hourly(), pr_cmor_e1hr(), "pr", "E1hr"), (prsn_era5_hourly(), prsn_cmor_e1hr(), "prsn", "E1hr"), + (era5_2d("monthly"), cmor_2d("Amon", "prw"), "prw", "Amon"), + (era5_2d("monthly"), cmor_2d("Amon", "ps"), "ps", "Amon"), (ptype_era5_hourly(), ptype_cmor_e1hr(), "ptype", "E1hr"), + ( + era5_3d("monthly"), + cmor_3d("Emon", "rainmxrat27"), + "rainmxrat27", + "Emon", + ), (rlds_era5_hourly(), rlds_cmor_e1hr(), "rlds", "E1hr"), + (rlns_era5_hourly(), rlns_cmor_e1hr(), "rlns", "E1hr"), + (rlus_era5_hourly(), rlus_cmor_e1hr(), "rlus", "E1hr"), (rls_era5_hourly(), rls_cmor_e1hr(), "rls", "E1hr"), (rsds_era5_hourly(), rsds_cmor_e1hr(), "rsds", "E1hr"), + (rsns_era5_hourly(), rsns_cmor_e1hr(), "rsns", "E1hr"), + (rsus_era5_hourly(), rsus_cmor_e1hr(), "rsus", "E1hr"), (rsdt_era5_hourly(), rsdt_cmor_e1hr(), "rsdt", "E1hr"), (rss_era5_hourly(), rss_cmor_e1hr(), "rss", "E1hr"), + (sftlf_era5(), sftlf_cmor_fx(), "sftlf", "fx"), + ( + era5_3d("monthly"), + cmor_3d("Emon", "snowmxrat27"), + "snowmxrat27", + "Emon", + ), (tas_era5_hourly(), tas_cmor_e1hr(), "tas", "E1hr"), (tas_era5_monthly(), tas_cmor_amon(), "tas", "Amon"), (tasmax_era5_hourly(), tasmax_cmor_e1hr(), "tasmax", "E1hr"), (tasmin_era5_hourly(), tasmin_cmor_e1hr(), "tasmin", "E1hr"), + (toz_era5_monthly(), cmor_2d("AERmon", "toz"), "toz", "AERmon"), (uas_era5_hourly(), uas_cmor_e1hr(), "uas", "E1hr"), (vas_era5_hourly(), vas_cmor_e1hr(), "vas", "E1hr"), (zg_era5_monthly(), zg_cmor_amon(), "zg", "Amon"), @@ -1139,3 +1495,61 @@ def test_cmorization(era5_cubes, cmor_cubes, var, mip): for coord in fixed_cube.coords(): print(coord) assert fixed_cube == cmor_cube + + +@pytest.fixture +def unstructured_grid_cubes(): + """Sample cubes with unstructured grid.""" + time = DimCoord( + [0.0, 31.0], standard_name="time", units="days since 1950-01-01" + ) + lat = AuxCoord( + [1.0, 1.0, -1.0, -1.0], standard_name="latitude", units="degrees_north" + ) + lon = AuxCoord( + [179.0, 180.0, 180.0, 179.0], + standard_name="longitude", + units="degrees_east", + ) + cube = Cube( + da.from_array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + standard_name="air_temperature", + units="K", + dim_coords_and_dims=[(time, 0)], + aux_coords_and_dims=[(lat, 1), (lon, 1)], + attributes={"GRIB_PARAM": (1, 1)}, + ) + return CubeList([cube]) + + +def test_unstructured_grid(unstructured_grid_cubes): + """Test processing unstructured data.""" + fixed_cubes = fix_metadata( + unstructured_grid_cubes, + "tas", + "native6", + "era5", + "Amon", + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + assert fixed_cube.shape == (2, 4) + + assert fixed_cube.coords("time", dim_coords=True) + assert fixed_cube.coord_dims("time") == (0,) + + assert fixed_cube.coords("latitude", dim_coords=False) + assert fixed_cube.coord_dims("latitude") == (1,) + lat = fixed_cube.coord("latitude") + np.testing.assert_allclose(lat.points, [1, 1, -1, -1]) + assert lat.bounds is None + + assert fixed_cube.coords("longitude", dim_coords=False) + assert fixed_cube.coord_dims("longitude") == (1,) + lon = fixed_cube.coord("longitude") + np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) + assert lon.bounds is None + + assert fixed_cube.attributes["GRIB_PARAM"] == "(1, 1)" diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index 4af47a44d7..43b9419f64 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -418,9 +418,6 @@ def test_fix_metadata_amon_ta_wrong_lat_units(self): with pytest.raises(CMORCheckError): cmor_check_metadata(fixed_cube, project, mip, short_name) - print(self.mock_debug.mock_calls) - print(self.mock_warning.mock_calls) - assert self.mock_debug.call_count == 3 assert self.mock_warning.call_count == 9 @@ -867,3 +864,30 @@ def test_fix_data_amon_tas(self): assert self.mock_debug.call_count == 0 assert self.mock_warning.call_count == 0 + + def test_fix_metadata_no_time_in_table(self): + """Test ``fix_data``.""" + short_name = "sftlf" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "fx" + cube = self.cubes_2d_latlon[0][0] + cube.units = "%" + cube.data = da.full(cube.shape, 1.0, dtype=cube.dtype) + + fixed_cubes = fix_metadata( + [cube], + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + assert fixed_cube.has_lazy_data() + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 6 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e32e3ca3fa..f6251a8bb0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -100,21 +100,35 @@ def _get_files(root_path, facets, tracking_id): return files, globs -@pytest.fixture -def patched_datafinder(tmp_path, monkeypatch): - def tracking_ids(i=0): - while True: - yield i - i += 1 +def _tracking_ids(i=0): + while True: + yield i + i += 1 - tracking_id = tracking_ids() + +def _get_find_files_func(path: Path, suffix: str = ".nc"): + tracking_id = _tracking_ids() def find_files(*, debug: bool = False, **facets): - files, file_globs = _get_files(tmp_path, facets, tracking_id) + files, file_globs = _get_files(path, facets, tracking_id) + files = [f.with_suffix(suffix) for f in files] + file_globs = [g.with_suffix(suffix) for g in file_globs] if debug: return files, file_globs return files + return find_files + + +@pytest.fixture +def patched_datafinder(tmp_path, monkeypatch): + find_files = _get_find_files_func(tmp_path) + monkeypatch.setattr(esmvalcore.local, "find_files", find_files) + + +@pytest.fixture +def patched_datafinder_grib(tmp_path, monkeypatch): + find_files = _get_find_files_func(tmp_path, suffix=".grib") monkeypatch.setattr(esmvalcore.local, "find_files", find_files) @@ -129,13 +143,7 @@ def patched_failing_datafinder(tmp_path, monkeypatch): Otherwise, return files just like `patched_datafinder`. """ - - def tracking_ids(i=0): - while True: - yield i - i += 1 - - tracking_id = tracking_ids() + tracking_id = _tracking_ids() def find_files(*, debug: bool = False, **facets): files, file_globs = _get_files(tmp_path, facets, tracking_id) diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index 4c76ba2651..e776b9caa2 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -4,12 +4,14 @@ import tempfile import unittest import warnings +from pathlib import Path import iris import numpy as np from iris.coords import DimCoord from iris.cube import Cube, CubeList +import esmvalcore from esmvalcore.preprocessor._io import load @@ -52,6 +54,24 @@ def test_load(self): (cube.coord("latitude").points == np.array([1, 2])).all() ) + def test_load_grib(self): + """Test loading a grib file.""" + grib_path = Path( + Path(esmvalcore.__file__).parents[1], + "tests", + "sample_data", + "iris-sample-data", + "polar_stereo.grib2", + ) + cubes = load(grib_path) + + assert len(cubes) == 1 + cube = cubes[0] + assert cube.standard_name == "air_temperature" + assert cube.units == "K" + assert cube.shape == (200, 247) + assert "source_file" in cube.attributes + def test_callback_remove_attributes(self): """Test callback remove unwanted attributes.""" attributes = ("history", "creation_date", "tracking_id", "comment") diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 90b4985a6e..3077901c33 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -3392,3 +3392,116 @@ def test_invalid_interpolate(tmp_path, patched_datafinder, session): get_recipe(tmp_path, content, session) assert str(exc.value) == INITIALIZATION_ERROR_MSG assert exc.value.failed_tasks[0].message == msg + + +def test_automatic_regrid_era5_nc(tmp_path, patched_datafinder, session): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" not in product.settings + + +def test_automatic_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" in product.settings + assert product.settings["regrid"] == { + "target_grid": "0.25x0.25", + "scheme": "linear", + } + + +def test_automatic_no_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3, automatic_regrid: false} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" not in product.settings + + +def test_automatic_already_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + preprocessors: + test_automatic_regrid_era5: + regrid: + target_grid: 1x1 + scheme: nearest + + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: test_automatic_regrid_era5 + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" in product.settings + assert product.settings["regrid"] == { + "target_grid": "1x1", + "scheme": "nearest", + } diff --git a/tests/sample_data/iris-sample-data/LICENSE b/tests/sample_data/iris-sample-data/LICENSE new file mode 100644 index 0000000000..6ab33c6548 --- /dev/null +++ b/tests/sample_data/iris-sample-data/LICENSE @@ -0,0 +1,10 @@ +Data in this directory is taken from iris-sample-data (https://github.com/SciTools/iris-sample-data). + +It is licensed under the following UK's Open Government Licence (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/): + + +(c) British Crown copyright, 2018. + +You may use and re-use the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. + +Any email enquiries regarding the use and re-use of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. diff --git a/tests/sample_data/iris-sample-data/polar_stereo.grib2 b/tests/sample_data/iris-sample-data/polar_stereo.grib2 new file mode 100644 index 0000000000000000000000000000000000000000..ab02a2d13fe920e78498d11f07291890729c344b GIT binary patch literal 25934 zcmY(q1FSGSur0c6+qP}nwr$(CZQHhO+cv*#?|uJs?#s)2ldhF^X3}YsrcEWTBq;;{ z008tK{u5KV{|FTjfDHfu1OO0x7Zl}xwEtri1pbc(-v1*Mu)qJm{HJ^S4-z^2Z~wQl|H|l~{~Liq z6#sfH02lxe1_0pif84-j0Bgj+23bj`2?k;J+CEwFMO9zux}o03Zt(SXemx zf74$A0M!3fY(n{;L-PEo3@+a-tt6FYcp_+oCiNXvsws|XN+#((KTBcNz#r6uc_Gp$ zT^GG*;n%atNP`I31ET$(L_?|TU1DpOST4fn{fL)Y6gjk1*NS50OP_uLg@z{5@T*kt z6K~*0J^pl4$GQ#Fql!%|wYjkwS===V}CSU_4KDf7Kt^J*v5( z`FZ`G;1Or%zKEk3wo}+adlh0lU=W_eAyOqjyk_l+E=e@Y@z9TX!NbD~?y9b-T)jE( za2H1j0gxzDo#xPbD>WxA{g+lVuyY|qCKJ7?FBu6{0SRY9J_btR`66EeI*`1D=VYIn zEsP$1NtnrKW-8+pnPp%%jxc`Ziz`Tr}{~`YHYun+M zRsugaLEMy~fjy5qWUFgFC)H$FT#~}faV%}Oz$GcOX#lOFsQGvCIj5x3<2Vh<68oeV z*q1jDD5q66IjXPJxVwwaLX0`eehOfS$E4kaB0%{M5;0uG};`k#SOKNdpKEl zS+C@jN~dzQC&7{3aEm^I!-Q!RUPFvFK=nG{ZKrrbP|SOlD(u+^B4w8b z!n~KbO^17@vajk);@F#-IYfo>z--lB?KXQa&?fz15do4njyckhhqdom^j4z4RfNAuy1mHns++8U2?Y`weBkrjF0ey2Y=n{D;A^=(X`jDJ#!3$iiA)0Of@OeB62 zmhtzvt)YMW;cTXWNAa}DMf{^n(~za}J+*fiwXqIv?_KkK%dCb+`jCMvT8X1Co^Doy z{RhlH(2&kGf7NY(z~v*qj}N>Llp-==Ml|y&8I|*NtknbPB!U~*soqH%={1jpb4zp( zDiKnuoK{HUTzL=w!H<_&@bk4E`1zWJu?y1UL=k%yIm~S4T>(sDV-}I)a^{2&p6H2G zq@x=dMVo=BHTI-BTQzDOOTH74v6*J`s)D25khZw`&HY-l1j-*I2S4BastL~ze+Ng3 zgS5VHgx=Ej*D_CML=R@Wp9szEm}FlA;Jn}Ltc_)@qH1?5A?VHY0pTKln;kW6bd?HD z1NB^xWJtIj(*iil45-p?zNk8HRkS_UfmZ_>k8a`?Pn zqhLLjXxy;zuFuOUZ++*gn=r^$+wu(ovdw8d^zm{UiQVT|&v_;Z0_8Uj{j`z*NKqD2 z)KWx@_+H3DGC^D3faV;wFM_JvjH{JHR)D)CBT(PslRpxx@H#RV;ZJH}Eku3;-m^=( zo~|S;ldpK2&|h^A=Y|{({c0K&4KR9Qa7{Y0%|gPY4B5c;`>QOQd3wkt1f(J@<%Yt{ z#BFY;wkRY8!00@OXp37JrsK@-YTNEB2*lytARk{e`l(y9%8OZ=Kf+Vu2k#PdOt^Js z*wBF|@!D05s-XE*4aBeaX)U&qpc@vt+jS9`#Bw7*QZs? zozUM4r@>Yq=+^pDOT7oRa%mJyO);m@`I=bME170eF(r5PYVnP%PZ>YC>o>xq+7r?$ zT(=@O*lhT*&XLDz_B;NAqlK_^X(XvQz+Q1>0r>-?*WPKL}W12U~@>eS*u+E#!4ga|z z0f+?jA10}Q-hQ@*2e^Med{Ydo&*tBrHxNz;8Zd8aL)|bp1pGbmLLUmd;^74i8kD9~ zx?gYNX@md=E0aR0IiLG6qx>NaZ_h={x$KB9cl1`#H(F5l?hcMCLjz3_#f!S>dC=kRv?nuTU z7C~tOx^GkP#mtP&Y!tq__+TD8#knwNzOAmERY0(BSJ+}qHNuq4st!C_Y6{FAji{x= zJG6;Umfvle+e-)D9rSJ59w1-B{>@Rb;d8hy$4C;p*ZH3;iS9pKks#Mac7W-POE)HH z26BT^$}EL`<}qjzvrN?T#aJOBX4bXXo%7gIq|D-Qb51$ync=h@qm{OS1@iq9&`gQ9 z)vnv)&PvBAW)?3lVh#$7t4!>$+}tQLwb(*xeNF50nLk%E6~t#p`L{7G)=hNh-cc71 z>L-?ji>^kBg1XX+>5t$o@a#!8)CCl=TM+PWqoC$&2+2F?#O?C+@5b15a~bk0KwD66Qtu*ri7m@OgLerPlNI!H+xw|<6!UKEzKTObN z3tSEbw1p($zX+R$*|C-WA>o1~W;(kRgaRh9h{nHdw}UkyWbcqDUf2$ZiOQCiG^`_A z&?E~HORNC?CwbwS;^-rl-GPuiP1z_<*z6GX2YmGjKYPPweD<&nM6{ZIf_7j-;AY0} zkUUa|HTHBN$sSjVMIQ-}h^i*w)$CdZzFQ&N@f?|>ZZ9DEMKm1ClYOT;-b9kW!f?Td zkYQ$yOz|EAu{0vdLoQB6UFECdf(5bm4O*q5a(@g3aDMsdkTjhF4~t}=_9&ifO0xpp zAOgG6!sZ6M$<{d?YI( z22mWw!3(0R7{Sgii-H+e0XgYuV7Cf?xl=wDY*F5Y>EcZd{vDzF6H|(&Wh=lgL>a8!cqM4O~*wV zYJ$)L0c)<75pRkd=ng)A7@sfLunUSzT6V|+SQb3sW^ucX6|dfzH{~8>-SoXKB}0xD zI?o$Of;0ptdwe&h&vNd^tDDQTFgp~uoesaVSpXu;OF=2ID)>^sF!p;uWjXM zzn(VesR}o&38v>D=cHsz^=s-Rf0lw5+_Drli&Yqt-|6sAkou2CoBh38Yb>h?#*f(P zlsU*wNN{?W>%m~u`ot)vC}!|HK*?COYHS}7OF0vfZ7VrXP37^p7rCrulCzE}tPq5g z_$lbgiSS)&e%e3}Ppwr#{tW3NN!~Y?M``gO7n+mbFH`SuA@MAnhfH$e@1aN0r9I~h zAnu^{gUPVUGf5o04Gut^TN@2d6I`}F6~fi@?TXaQw+s|h8niMm!n^YOxbB(HRBwO^ z>qpcPe(RM9xNzFVB!- zDo(>7%SM4N6d{0o_hzt9d1$wAA4_uJnKvwC(^O7bCgiAMC9^@#0T4!g@vJuGYC%3m z3JZZjvS~19aCx=MssqI8|@k{1dx-DFpU!wghx-hZ_)9{d`;~ zeWdi7hPbI?X0r({HTb0x2F>i6mqkL3wv~E!h@rb&X#-l9a!wP>q_?iAKz0%v*HU&OH@m!;p zx`GKcFVd@zK3_JAq(=bw1`sdjUsbP)nc?3q=H4SM^?4_-8Q+(%=P{?I2By4E90O1Y z5CftX4HeWSQ;G@=Q{s?A6SLL_JXz&-1$rtK_$L+YV&MacHJ7B;yO1VI;`p1<<9W-;`(D=G`S$(V`K-iFDj-FVdvvQh_!@6m8*j5)VPyWpi zcvD(i8Tzu7nS_`4C%S&%y* zyA@J|{mT{G&nGjsaTmeWk~K^{-fziAl)b-A(H*k}a?rIXt!A={{MlT>X;m%C?az)j z`*~4B5NHx3%qAM(_&QC|!Y9fXpEZRO!TpU5Yv5cH;`*{SU7#CtnWsQO?^}~?|5)@o z3;W2=!KEZizflRNgK3e3kDXrv*5+n|#-H;qL_hW_2spm{UpH@cQ~v41&kF?n%#f$f zj-Yd8`avs0qS6gi8a(+DG-$&1NmK=gGyBz(F`Vt4ucQblAMv_4h}eYKSHR2@|3*lW zUAtFlrBg@OvE^hD-|tms25Nbp10^YoBD^i|Gl#$DUCX?Hwn5uCAf4q3yFlq%P{_*q zccxLC$k27W+oasfo|WzLwNCV@hYBRM=C&q=ODcjS|~X9iB~u>!=NA zH%keD%2snEt-aMFo(L&9XelIoo$9IZN8-xpz0Tcl1H}VEccegm=d-7BA+CZ^jV(eo zQCRtPmwVGup!i=EoRwhc<@yV*o~17($fK+?E4^$=aChLrnrx$-H1b!{8h9H9z-0V7 zdbI!S88s`;FNd+=fk+;5h@zcp2{mWF`clmElVQ*yG_egV$ea*2_~5BizHIn*+ zDqtNn+K)U9(oC_mFL`T-J*jw|Pt<<8Y$@BINxwCIDyS`FlA)Yq70 zO5VU;d049wU4g$Kbu9s@GTJhbb*p?lna{}VZIJVC`*j1P^m&kl9lH{>BkJ`0PNRFg zzmel`*^So+fH4^IfFzD&_yaLf5JkU#*`%w8mgcf@3BRG$8T0G_Q!)6N`i`!pFG*dU z)qC2JFc)PsRdXxXpveKTorUR3$=K0A_uw`2AvpArEKNn#EA?#;*>TZFscS&xSgf}g zo~-#`7^q0QB?gZ?K}bt9ree!&YVOAbht0 ze~AoT9lwek(xh{h56aTH4Hs{80BM2PoJ3qiFL0AkTq|Vd?yd9d>qA+Jy)0w^YccOv zv@CZEiOQfI_Ez$vD_{i)P)&DfJb&?(TOe;Hz*&sQ5w7;7%y(4%ay&_iw&L)|gC$gsz5UY)TttLv zndY{1GqnUlr8QY3Mh#g_N5n$cN_SI8GkAW5ov&yb^s#wgi8 zS=1KC5GES0Q8bZl7vO2CCF6M(b~kIj;E8$4lXqh86nSVf)yMCtb^<)2@H}n^%R$|^ z&K)W0PYSnY93HXeXoVpgc9TV>NZso+KJ5%2_YnFo&KSLsxEpn(fpbTG7V95iz!>B3 zbv^lF#0uYX6nN(Y$E$EHA|~>4!M?OD&FQaw;iB1kQIE_?24QACHa^c@eVF5(o{c z#+HS600O(6-ZpdF!E~8wuMtl+@)IP z5|%JefKX*jZ@UkYdJ1Bs4xpAwIKzwDB#MkCKSdRY=5=2~f%9c>Iyhb2?j_tC& zYuAe~*Bcr>`i+pB7lx3CakgyZxm0h^NO8GN+&KKySRw$dJy;VX4&}M_il+=_thj`$ zQJ*G0s>{1)oT&I;{`p_W&2l&|##L%>8tcagSr4@EkVpSpn_s6{tPVnmGEinuEvi|& zJ!Y$AEZ=&(M2fLuU@DzA=tcX(OQQ!8Ek3;YxZeUI6Ib)%Lp^F6T&!XnuEYce1=iSf zi8qEp%SLqy^%8o~6kg|{_|ay@lviSkGo^#jdojxl^)jDB#&--r(fYM;Z%Wd8kmyKX zuwA&Z@w;j1aqj2boHJA$5PJJ^#F-v_)`$o_D9R}$9=5iXaOODnJ(G26BfJ>ml3q&4 zj}|U)m{<08sA=AsAP5x&R1Xk{^vzYr@0JWR6{d(*g@;H|DSGt!GT+qT+RZ{@iF{xq z65X7{g7C$6TSkcJDgk~}y~4R8>NEL(l<-}Jaz9dovYWKh>O4-BCO#GkIV4X0J z>Vx2LKdd}JQ-quB%Kq;rWeZtM4WkGffR6&9bQ5YWSwjuKRSBd^ z;zJ0uXm|is$r(@U0RszXLd(+~DbXdB3^(SZ+nq#Th%MB#? z>$Nb+&lbi<_pa&PeAbfyH)hxXRU+tjT1En!3z^qN?ft_~SnU8f#y^MzZ-g0N``yEa_3w5B7O6A+ zK80=>(u#e7tOCCRSLZm!peGgZGpiUAluMq~o{ZGDxd0?3d_C(3^C0^>mxp!|vpIKN zApO(Y%NG|)(P13H50;?wu*M2XS8i+MsS@A(H3wQvbbQu(mJa3k6`uDReZlq?9ak`5 z#}gK8`->ITRz7UUXrQP|2^ZI=iZ~Mgc)M<4eQ(G&5jn&>$5zqPBw{x8M1Tj+WU~Vm z41Pyr%`J7peYeG@N}3G800lcQD_(q8*&&(Ru(w8hPDd|#)faNjOnQ4jDR#>#wGYZu zsBE{xIPka@MOD_YYyIA!QeF&XiXv%oU(z2{Jy*$fsVQ{&_#Z^nhR<)hn~algwk+%B zU|CD;#dKPs)V{nkF)iemn~|*A)ucwx^IND#litjjG+UB(OAfgVB<4laB4>QuyuCX} zO0$(ks>PRnvBCq~d1|b=ToBBCca>=O*ML74ocvJ}Gg9PN?ECvZKvl=QT`jS^x+s(& zeJuglBacTc>QzF@Aa1Rh7Q`!p+1s%X`_%!dH=G_hc*Di8$|2$V9fOpAp|%7D;eM4n z@zziPua9_t{t|84j#crep&CL+*Uumo=e}_VHQN#kIv;obd8`M4r`~wO?X#C4I+`y{ zyY9Eeh1hcnGmA9@yJcP=GNuSQ89*^qcrdT0DZ-enGQ6k1PqvP=_@`$vn0@ccf}_$EhsuF$Kz z&UdA-)oR@VW$93H3|7l$b|8R9y#()G@AVvbxO!@F1>QonC$GzLd5nHX-Gkm~OJ`O( z|Ay-mCV+eo^qir1C({UN;?9U4mS|wYaI`J$fl;6zBG2E?)mGgeP`e#?))HA-OJwQ3 z|J#!%u_3#zKA%ON&}>t*d$FkPE|D?N!Ew$^Pp>~D(p3RQD9~pca_xa=bzeVeu~7Lc z))M-~!dWFtf*LVju^}Mwa;cxB@ni>CpAB&Z1DH>J!Hs!g$yRe;WI`X$0TdG^nYz-as%-bpSr*a}uontL0}LPS_UOp$|L_hMlH zc)MVF6WVu5dbF}Ss80+N4&8Fw7!yyBlT@os)1#8Wo*04KifQbtDE0~mK}RCaQKxI*0n zqkVb`ZVhqyB72=g-qhR2;?Sak+2@Q&Oj_LQ8y654P~QfHC_M&_MMO7yzG@#n#7H~*dehHqlH~1EM}Jn!YJNS`g1sZx!j4}4pcIqBdTC$0zDMRULy6TKutX{_ksU- zXFlqu1H8F#x6TN7H%g30H6*}c@m`WQcPd>8iX~*SK+5z{lDa1BSB{oB1nW^W31DzY2*vJb%fE>XZv6_J%g zA=xRbDyY#>nqYB<7Xvgzb&3*%NEsbS?ZwojGW~TI*nqsQ|68w)Pjhv*8ba3l;_*Sv zb!0$ySaQ*mt5!(!%_pA3d(~Ofi~f|FL?KU-89NyY<7QbP$=KTXhc~Q(w6?;whvnX- zbWPqRCfZ0>?i5mKOVmH|US_3YXSgvZ%_9>@iTq{g^Rkm(s9CqGU%l%HJpY9F6Vnd4 zI_#hlphl77OU)%z>3v!H;}sF&6WH8ESO3f_?4c3CPgr*b95xh99pYbx(RctX)Ency z8+S~CEV(&Mu+sWYa2t~ZSFsLroyl;&pJLgPbs+RV^UTLmlY4{QH*D6MC5rh0ya9Rs92?f-5<^K+;-X(g{ zJm$Bm8XWEyv*u=j5mjZ#*>bdFx+P#MG)Xd`nw0SAuBf{!>o37*%hLC14r z0|4`>yhLG{4k(eel2m#t`#P0NA1e^@gw^m zguaR+_j_<_hkh#;NME7t%)g=Wrl~rH`~zz!u$H~8_`Or-HKCi5EV;%7pBx$_`Jto9 zB$&K`N{nMpxMzxn_fJyXWQRFYzGEx*suki?mh@>Phm3zG4|JjXuk*ir(R>EpM^>Lc z3ho8qTjXD=-Bc*@okJq_(9;>uE?YBvpjY*N^QHS(xY3*`J#|idtr%avD32|i&CUna zyv&&~&-J)+@UK^i?aLG`w| z2VV0tVnayn11ZH%XT5=d+YFOG^m46&1RLzHR2zz1Pw`V+W}(ER=62N@102ISh%UpL z2HVO+%DY!IFxbM~-mPm$Drr+h;XIYw5^ zbEu~kM0i%)!FJ!p&Fs@6+B;~ICX?7gz!t2FZX9(sK_nFgj>QVF$Hy9Qx{qSPH4Lvh z7OyOI>>MXM#=~}iJo7yr*ooYK8(JVzU>gb{H$(R6K~PaUmc0~Rp?SZ5wVlBHs+fm` z9b#4l^J662oIWA6S4#M+nPs1QD?*H{d3E{N?l9g z&6oHwRE321ktGovI$rGmLR*Sf$dZ(pZC5H%f6+~L-%%%gboXfd4MiZ%Y8n5x?Pd`g z!4!BrK8bJDRk7miXq(97OLyvev0yGmh%K5za{HxJURrT>padO{1A+oYM^;}2JejJk z4d8qUMiPkgrU6lDh6Y$jJbhVkF|<+l+Tsc%Sj>kekecTMC!*2lG;o_3ho*`8@C$Qh zhoiWL5K-G|K=`DUE6i2Cja2}JNCeeM`FTv$eo8FhCvPDoRD5- zpkU$}KQCv6CnS~&QgDe{FR;B5J>W*Mk#hXZ)$7L=XFgz?CEYbEew@~3TlXECwC(Bk zQR6tR)wIO7ccKNc>sP!+Y?U(?_*zls`Xp{<*=O@s{Ml7}Pm{%J|4hexK6`r-ILshe z5FWB-=FAVAfSG84TWOu3f-^aHsh{E+RXa;ro7FMy#-^Auvw_4F^qvKAPCs(fOYTuc zE?zwrzPrx8z6Ie)nWqj2<&Uozv|$8krl2d@Xjv;VXl=2p;#P|-EZ>2r5q5(J_hsI= zN6TEz>xMDv{K_3^m>=1niSBbRQcm`Ujh2Lh$*bXT#lz9?7Kzr}Q{~halkrOpN^`R^ zaD9YE?)$KUfkfS z55h%YaSipd>cQj@`goPqTru$N+NiGhC( z&RPv4EEI2gh~v$E2Q#6NUJpvD93>;}j1z*Ya0o4q8`(AEEEu}r{p(8MV6eoTS|F*F zpim2qbYsHo)?l<~Fyg@%>&EU;EhyPzp>Qk}<>CjDg~g*3$}f^F8v{9I1lSeLmlude z05#*i{@>Bqhk^A;0X!l+)BCTr`W1db#(XbU#b|N}?G{-N14{H|x8wmp2Fx^Tf~5XiKZGHbF7W__2oxo~5W3ZVE@k{2ZeZ?dz^>Lj0$Cnpq*Hb%Dc5m%m~&cvfPx zQsW>`=*|#r42Nu4QJPj82%?NPS~LT6u?*e=bY8j#i942G;NA6P%7_eb9)mkJEHbL_tVRK0?c@cPL0-5>eSgfF`^#+zFfUnbVJ0Csyt; z#pPB0l50=-So#(L+~wnmqz2;(!&}v~3`sqRVl+3@i6dCOxOy$V9R&zt83iG0nC>cQ z>`vRB222N+l1YJ3v|ZD??QSQ(B~ef(!PS2!955e!*hBg^5JHo6lmxrT@zaAz*@2_{Hq0?$GYIAt^f`JZv{{f)JmBWnO9H|505Bv0NoW+z;@8@C|Wc_ z38C21f^;R3lDl0$M9dpCAqL9LFoQ?neD9@#V0fhM-p+5uOtz;tyG5cKzDC2hTt|W? z(-P~i4Y?(i@pW&^GE1|!M9AAS31&tOo-Hj9-=F3c7YAlRxzlm*kuwK4Hcv;a`YWG* z^5c(Yn{6jwH&fsKKU!(ZPRO3uIpo?pG~&z`T~FsyPgDRX1HRcIUogN1q$|L>Qc5#Zh0_rU{3=A}k!yliSqYY9^i@E*Zo67~P}x7nfQ% z0ik;ea|oBZ8EplbcEf=RG*>Y`PlarI0G)-MRkBN)&e3Cr9?IO;s&t1LlDycM!pHq> z-=88oXc!$2gP6dD$)@a?y~3tkyEz6urDV(G<52C)IUph{uB#dg9B|S<*W@eJTaA`Y zfWjx4>=qwr(x>CF>mc_6QFexl@+|Y&!oF9M^~)rZCA0|jgrv^Uj^62ZJk6SO#=Cpl z8BtQBL}!kOoBJ6ode2XLcaEP(!PA@UZ#Hpk%@5IMm{QbYJvsqnLS4jW^)AiiJd zaN|f)B&PAgqwDZTws>Ha43t$N{nZP>Z{m1(X^HgsOpa+QeuKf;b4;-N2fRNnGFgvK zx}#~gI?r?^%$V)E(7HEt5N7UF^wF0m$ffay;B(%{DiL)p{&^>XXA(_CjvjjdMA;8B zxXwj^s|sx{A|$tVu6XH;9ot1_zA;OV;j!Bsgyr9C~gvAf&6<5um zNe*=9q^C114ev z7c-@sh(X;jB`S{Ah#yewT~%B(MEh8tbn_D*$0$c*

@*iEWsdr>GVDt}UbgL%&3=4@oRzS$%>W&B|W$JEY~$ z3r}A7Rrrf6MxG_BI4qzfb}FTyV%=}-0_ z|HEh$>h(|H>ZG;^#c0;>10Ij7 zp?MDKL?A3LX#hGDNM&DZKD_qSb`@h;Cs|1YWz*&XU+FvhGJ9Pcz5>f4etyN3nUv}e z(vuq|-KaDLaK*zs_wha6BULq5BK@RJGF(QiTzi?%J<$V7qURkHlBNgu{c~iN0vhwO z{zdNYv%bL`olC8Ex|0$0AmkQZY+Io(MLsiyS=tnU8nkAAcSd<|?|O(8;0*+3RQnyg zxredtaiF`N3Waj(yZrEFbM$k{5eO)FzhgQCTxx{p4f;T!+qna7hk&=6kt>X8n0Wf+ zK@0+JTra3Ol=61|_TgIpw#xDmV=i#OwuAnuI=9e_j9)A=O)(<&Y4)R5w!g-$Yn4T% z=G=sk*2#>R6u$9SDRpHKD;8&pyPM#y$Z|RZI(D-!4NaI$?LHO~0V*q(Dw4{hWm?G` z;$JIR6VZw8dx?<%QAF!8pIkv$h)8ggRfA0UB-?7kkPD0lvZ`hg1RPcL%=gQ%ZLEEg z_1o(V*ZgFC>zu2wD^Hm5r>mNHNGmXd2OU}+xUmZp>(Vki7n23!US~3jy>zpD^>77S zFS@nyIu^iI;vMiJnfPiqY_hO_tSP{j*g;SB11D!H5L3*m;&bf>sdH}dED0aIUhJ=L zpJ32C2P`LtHpi3zgglO9$*|(lU5NMNpWM~ifwJ0UYwL_Yh|@AAj4d`l#fStUKfN)nT^PT ze%`K7+L0J3AW5G}XQP_o<R~J~t-NBFU8$ z-LtvAOCZ1zV-gqLz;iQwiZQTEVSEoE@Hi87i*!*h024nnn5&QYj*2z{SF7A87tjDs z8j0Bj9N_qX8_;R<%KOgEd|EA>u(K<>nsqwF9dod8lu)9W{=+CfrO4dhI*sdL;gxUCiJP z{96bU7si9gWMyNobYOR=G~*2(BAGxiZf7L9RV58ge{Nw|Hvh@@e`HI$FjSs%Ql}k8 zq(iu?1wF~WePC<_jB0Aj@%eM-PTDYPqM}fqtSZ&s^O0=)CLx{jS)!?&F}VqnO$E+7 z&~)yY9V|VFkHcjGM4PVVE5=aylcpGH*4G$!WY`8c*J_>0r1SH$(&XYk~G| z6^rli9+~?7sFIC7NGzW-1zEDmye_Hwzrl(WCzIDL*ozM`;r-NJ#FJQw)?d4 zr>YL@8rnLiikl^Ke7F|F1=59gyh0z`A5oSwmI_k9Lz4Uc@u&9VY1ZiVvgcxZ&0YoX$nHwQbZsvpJ={UC}56|-{=1>qW6Kabf zM?)%<-E^_(C9raT<-lGoa)`iyxsL^xB$uzmWb+=J^O@XQ65<8dl?s3P=LH5XcB#0e zopAcN_V{2@(oXe=!ruj8B}wjUu~~}SzK5_ZZAK`315rd1o7^dgn!EmoJxn9SUhEBa zWsk|{1t&Rs7raR2q$xo{yI^nV(7481JTEjmJ)Xy%8%!=KT2Lj)j;6-jb1#;j=7q89)Oy*SgxZ&!OY&gRo?-{vUr* z~`WEDdmK5NPtP7cO zq^v(EDi`FItN;Ui*TdJ%Q7@8;(5O9+*ge^!?UKcE!?hMBjyQCHxhMak^BL#~fySKg zHHGZdyQR$NCFUKWmNM^l7W7oaAt0zpU?1OfLy6zF*TcZvK7Rk2qm;WC)}P}3PTdU~ zC=sQk7DVjnpa0Znu-99RI-OPz*_|_Y_MkE{5e)f~8d#4ECkmJu4@iKd+lVOzH$XNg z>te1%b=`ei_`SmI-bcL#-zj?cD1HIFPdMK93BQa|Hskt+glp*yRsj< zwPt-M`}@FW!s1Gr*HhUO^TfaWYyXk2;I$a{f5<*x)p)YdauNb^VMqqpd0;xv(}kt?8vEI zoflmQh<%w9QAaDPewP^#Q72nr>kitS!MTr_R&H-K%cseekn2?$fslE57s)&0wr8dC zE<6zfPVm!_xO2z^Y6rnNBTNi~m8Rp~33XV$P$(q-RFLs=?x3)fT#;$X732z#2tv^I zNxWQVsUaw4(^<~7*v<43O3FD`2WoG7Geap@_-f9R>Z1h3=$L0JTSRFpt~f!&)<6l? z`~9DZi#ly%)iH$A4(03j2wCstOvw4W>1Wv07djlSxHKYV7dq4`mYWm6yUIpRKlTp{ zlH(-0D+eufrbXsx*hS`Fic=d7#3o}x7!)+Qxw#=frC+SCUL8~auq3=k^W^HS1Q`4Y zOwxlriwF>C*r8~emCALUQH3jp#ZlHAQGM0m9Ri-fD;TL0*d&NF zh5Syx`=XatP@QcnX?U8SnTW6#4H_RBTltmMVVbTYwug`P;hLUqZ%OLh zZrb5K%Yvp%BYHIb^GLD_X`;@aAZ49eKezO5X^%wG((~wpB2tZB^BHxo@Ct^0#(aab zlzV&*ZF&bGhr(W{b@#FQRD zEg^GbH+%GeI7MY>vj%(7?*UpusP#!O{vBdW{JRuQ*L|7lN}{>W!^VUB5YW9G==(FG z1IpgIz8idrgwC*Iy$Po8re5d94@=$COrux@L&#GlrX3NJ{G*k8Ui2c6iY$X;7c zqzO7x2S3bF-*7KrrPVy$H|UP$LM@l;r@fegbYY>~3g&uoCjii5KHszF@$-^D0Ln$= z5hXX+Yk5>F!r?o|R{sSkcBvZJhSp{vb_mnxw+Qon^AE0~xFF3x^_Q`EwUTMww z@F#pEUzObf6GU@8n_$D4`1G-yjrjfVZ`XKPlxEgr9mj@ci;Nenn$4`A(+p8uIxn_g zCkOWsRW++c5X?i=`Lx0Z6xp& z;QCnzNAso(^*TB~!|bBAGARbwRzY>Xo)Acg#NcC}az?WCzd5vNW})U}72i;e7+Uae z;Bm|ld~`$#E9u?t=cV3K^kbnM*nq7hE$hZ%p4L-<`54)pe)OI+* zWhC+4$rSh`cC9>AszLNnH~gjR6!3;(!BqM~xob&eZ#O&zRf5MfHA$B{IPZ=aOs_wX z8NepYkx5`X0{$cM2Kne&MST` zuhf^5;Trh?2b{<}I^z={KadG`fVBGf?OIw{ZaiCPZ36B-y?0O+xx51=B-Vu%#TR1; zk`E>VtPQ{%LmrA)V9209;u|^b{Ok8pX9yNT7xRaF^@ZrFEJd4Fcrw|06x0ySN^7ub}G5BE1F5jm9I+hqU~AkGXQs*=@RJR zYRQu+rG%0O4^?iQjlk-JnazWhMs4474|6z)L@XO8nf$rv$5GERUs$=vO`l*=gK-tz zUKKhhYz}=`m}cKS9royAqa~Z35X?At>7GR;>FTHh4|50ZG5xM=FL=d(G5L z=VJUo6*aC>nvVg@*cRD`SW;C^;sQ71++`jUg+g!T4>V&PgW24`1wm@P(B-GSlAS!G z*Jpk1oj3;-sMHVBU=9m?BOSMrR{>gft-lW9S#cXUylehjzRIP6Ytf zlaHq@U#{EW64xap-ge#m85p>G#hBxjk~04X;qis&E!4Gqa7<3OID;rM_bLH*61T#B z>u}}WrA9`~Mq65u&U$_cyO2;XpF5dlZ0$OXWe z&{0YeZRpy(6h`qSsAfn5eS0;|MAlKydj$4Wsy5Or8tk?}B99I!T|_+g7d(^TdYe7Z zC(Kg`*Lg_w1IG`p6lE42yyn+}FnXx2kRwOR^V=*Y1mfn6P{Br?*8o(&T&J92N9WUv z&*6bX=eIq`LDYYeQ_>29%X=yCedc`G7h>-7GMrN3GX7Nz~y$BPnVChNn@vNN;C0>wv$Syp)iL zBW}j*M!WaJs?fMM<-i-O@=*}HA;`gLdjUuHR0gv}>HmYt8!Fr_-GIHscg3VlN1Z&H z#FrN0mfclqr4Aly1uSR{@a=-CE`MAgH|)+1U;2_pB~9_(&6Qz%N+ zK=vIVr@)C@W=p(ZdiFAmU^vh$I#M%l!pL0ol#83lWA*&HiOc%V@qT`W?s`p2DFd0! zgbNKx^6c}ZuW-f_3i#4362G;zs(sf-gn7ssgpck9fK#KGYb51ZA+@FuGV6c zr3%=-o3~hC6zSZ$cQG6VZh;zdI_bk7e?e!OU)nw)iWGp99RN2r@2VNWK6(tKRGjvy z)vy`|q}LVTkp>tA#S$OC2_9(27AvhbpNI2KYyp1bzh>{Vl)HKa&rXUt_;7(ayb3+5 z=Pz*K@a{=)?t>lOtJ8Gn4euyQcSlCmDN#)HLjs@&_s|7Q-cXDFxIX*9It)Rcp3d%} z=wt=7SlQv&Uw(AOD?-q(XcmqK+VJ?dURJ}0-*zA*8n*f4!%UDywdumqIc@0)D@=7F zL97*Y%!*gA1~9CpV|Nc0ALCanJ#edEbvUth51gTZODX?4iO9lZ2fA23&ig7}+pSRC zO0X|NpR!~_l<*{Hlsj$TROV$dtNAVbngEV52J?a;k2|eePbnC=U?*o*LS1q}JqoEY zwRg7vOx4&B`9|9p{85MEpP>t z_O$Q^N=7b~MH&>3WZ8HhJU>isiQ|DX!`32|XM^`KWZCrY7CMk&g#r#2I8G+WHclw_!_`x)oWMzEWowt0Pf%poJ7sfa1N9fqh%W-cN^F_O`t|Gu zu1r;L!tkSO`@mNdhi7xAlk~ap5XIIBJXFmoyCXFcHUB4IH0K2=N*5z}XZS{vnbT+! zn5c~xIZc2X%!EeL>ivYdQeoA%TSvCik;!wW zVzA!~f;-AadgwtX+=>#^;<5tDSDh8dspMT9z8>%5(R@yL?huSpmrBu_PJbIk>Hr&u z6C^Ter9S4db~~jH>}fk(iIhY7wQQ^bPJniOsn#Tq2X9kOc``0Oo`IV&WT6aSA~q#3 zjYESKpYFPmi!p*3_f|kPJ9~#ozsskdiAwvjb#OE*6JPin*Tw$^?ZQ)f5sw8onw@%~ z>SoJ+{eG;Nqy7QRiH--#+}`Dj7DWlXQM$hv?}!4&rHS*`NnqKg(5&Ake2vyt{OdFr zL{FC1F}^0G+u>4Cfsi50;8gEw@5k2)lY)4t4wq zNw?a>Q*sSaz>&#|9`GWMf(aC@{Wm`8a7 z?6+yYplic87AFH^BC%EoVYFql(vcVr zP{y;a{JOD&11oD$Af<1!KdEBA1ZlZzo;(Tc0xxM1*^^6wEuQ|-YLCxPDC%5p{JAFGB@I!{*x3L2_nHIOR66y8(gE6zvRC%VYZ`N03 zeM-SE;5Bjc zH!F>H2KI#XFy@@*{!Xt$MDPRQ%VSP|UKv7*#K{Sm++a?6(R>?NPk5J}<%KYdV$p2r zKMjiEbw2A(!x7D3`?D8|vc%pO-rG<79cW)eGN0Z+eE@wtaWVePXsjHr@ymcyXr60i zDJNtpc2nqk88SjAB#!Qb$U!Iqcon14^TGmKBJpLdIV}*(!TfM330cn}1_e61Eic;G z>xO|CU2hUaqF%&e_X4#Mx_^vmz$|;74~&ve^{nMGqWtNV+0}xjqmtZg##6S8c9vcuoKV$rlakrD!+G@{MPUP6kPSwPXg8zb<#6y=N# zCL?&<(E*s)|5S0VPzF0vrjsL=6rFQaD}1p8{WhYOP_p?T+WpOsJF8}OC@lw%NviEb z0$n|cio6}&Y$NjivfJPcR_bnmblq|`2b0$91%l_}P<23oIBc9n(au}zX>a=e4o zt*fq8uSM7MeirsWajB{j;j1ecRKoF;o0s?UPt0PYiPWzSwti+@0UbuG_KK$1v4DZ@ zcF@>h;$SSd6zq`%MAiR2WcL0`u^N_k$i%4of{_3vpu5I^M6iVi2N>NECDKL2W*ovV zJh(RnkISuT#+m1wk_xlc@rB8ce}fnX>iP6Vc{`xfJW#ia`?lI0Y{G?7@9AF*n&{9I zp^;y7@a3gu1{7mlKBADS4vFpn=H3H0jD>^XSLm##>h_70SzmHVeNtzWDfQ$jaqSch z$X%!4S>k9Jkex2#MQq|#HcOT*Q);7q9QYxSzvUwVb9>}=Z0{X%uTR>=dmRX~=p~LY zO30!o!1}qY`*4-dK;?Mj;R zJRjBV5ul9xG~Mg06U~=8KGH7fnrisr*zz_yaci{>oUNMjmQL(?g3R|^e~Bj=XOyg` zEmJ#$;MBh3tE|}2k#6iMnoy!gC_-Sjxz6XtUD*vK$KcJux|_0s(RJei}ts+DJwQvUA`0(uP{`@+S0yIL;CHm!A# z0>X2o2VFi|rCEV^vMK=Bh|KwR!ggU8U##uKhu4l0R-#9eu39t*={O@3sVDRuhdK__TQjN0rf>KSPBLx%;5s_^MGH zWLg|@Qlq=Xp)>uh-nLr+T{C0qA#m$c<&0*r?qW41<@8>K6QMHp7oTcmR7~*}@M|5G z7g-+FLOkhc4mq*e&%d7H+2hat!!Or8-^>?2Jj=exq%hloOMo_uk^ERM@YFm=699o) z1*C|AkCNuDJhRBwN&b}Ri0WM|U+P-%WMs{A%7jbOs>{%(!j8+C2NGJS2Ff$U?zJPi zy{H}NtN(D4jiRu6VpnDOZ(0bw+67nX>*nM9#ykCh_12B$;IkLuakI%%^qH*SNd>dw zMjBU@Sq~3D9E9LwM7r3A(ZqA1_~Y!h!Upr;*xvyY@Usn!(ncaEpVHWPNzI5tOr|7G zSuy_;WL$b03mDngJBcpdTi~wktXBFlVn^)IzpU#ZEpBV;`;^y8xz68AdTXOA_wwcj zvb?epr_du9v`r-}vQL)P^(Y~3jv0qq&4Yo+n^aU=GA3+YA1pJvvF3zxL$Ta&`4c^YhOhZqUv_q`nIKw866cOe-!ndLA|+3AO#*em>et~fhH z9w#1(`24QY);Cd=;6pFOm1!0gbw+%zMlvlw8I~Nn$z)rnd?SfM=_wFU2A81ksmy;E zf$$%y3GP>YO6RJ<-`3zxsphv?@4{7fIt2OsrAt&G3lvW0UQ0=uAq`^MCx`SxZT{(PhBZW@QW+EhuOL*zPocN=_Ar zE|Ne67pdo9g|{L-x##Zd2!x@wN%g4`dlmK;Qa)b~qZ)ddLT^*7?GOU6IH?=rrx)z1 zgIv#KWPa7%pYMjPVEXz)NLETup3Rra1;qKFUFcI(M~e-DDVL64E+e8L71l~gmBnKg!#_knU1}=Wt9g7EtzSs8+H*(8} z)ntw6`W}9f>v&RAa>}HS3oZJ<;?I~N(hX{lKukm4X8PSgN2oUuyArFd#eb7pr?k^& zxWy%ZD*ih{)Kw90upj>o?bH^a@^SbXEe1A09(V6Iu~W0bEBBlRonY_gLjn$Ioh^JU zTR*EzrI{XB0W(6b|Cl?0v!SjJmCRG)v|jc~iw4a4(wo z>@l{@$seeam!YCj^QY0mr11NNO0NMLjMP=kNv3hvXc?KZ95J~W#Pf^&bSyfM);6_J z2cN3#9J&JKf(tpm^P?n>?6nEV1^|eiNKcpf6+7i%xhXgmz)@tTbrSXR{T*xFZRQ-a zxj`96YX!JLN_DS}YkWc1M}oBB5;Lm*Qb2eYf_hoOhpHajC&=6{;5z(mP#PhHKL?>d zh(sK#?17&B?uiu|*9M{TSQ&*|FTo#4JfJ;Bk;2hZ$oKC~EUT$*!7fWPpg-=?&*mYy z#Ys=&A@G=&v-!9oECv${wR&8G(v>bU{T@CAL)kXOv;sm)MKpTO^*ht^?rbsX&$^Zp z4DnB^l+lon3K-;m;3)kb{~RQjMU~sdZqAWi|74urZO|Q(uxr|ANPo2qR@Oin7z5!lBkN&~ zNgv;^7@!6nLj^}lj_@bBQ?p7s8OD`RhSq84be|NKUzWwpp*Ty(|dNrxp@gq0j zF{!KNqiq}BGL@U)(^f`@)9&~3NoeZ3`JrR%IDxg!Xxlb=Y|9zci0!5^(PGvXSvR+u zuu2O3n+Mf=C5VA5^wXSX3|yT9_WfkBXLU+{-iyI0LFwn{@^dqiG+S!r{E8I&98+4R zTs=y9;^L54g~_NL4eF<_K9y5KOl~T)f9Mpja0VmKm_24wh`)Qnq2uQ@UZ&QvAzXBu zFn?QBTC2g~CRKoaZu7s92Q9YRV>PIr3jFn>>n2XvQM`a;w6qu`SRe3wuHTmykRKM7!%Sq2jFS-InR)KGL7Y^IViLdH(61;R zS5CYOAcHWi{bU)i_4oa^OU4@R)^xEb|07L3s@g|V*v(kt&f!KdkT#P3Gnpixa z6FM{;nEop;Thx3u>od)1%@INk`cfLz51O+lC80=_)NqP=vCeEShFSBvgXn7xAX0sQ(GyldF?^JHSRcn_q3kea6W8m2n?w zhdPCPC(nOTj9(%EQdI%oYmeQ$omdG53tFNx{~w{4=5yxfmzlNZf26B`obi9eu;W)+ zN?Wiu9{j{B&m~om5)=Jc1xf<5zfK)gY_UEVOPxX)qxU;Up|c#WNz2$y3}{X4Zmw|6 zp^6|a*$u0xF3U2ynh#-#35#$#dv;6bLdGqI$q|LF$pbBz6MWfVhQ4F~oyx5nQ)29d zb<kd9D}p4SU5vu5S8&(u+&_t-^hX4q@Ew)N`w;vaxzMr37> zd-tt6BtH8gv34;@t{mTy{;f86B%uYkVm?r-gDOCN3q{{ZSs_h=tf|cJb@EYw}y%SZ2Q^X-C4^@(Z4ci<+Th(w~#r z_FMLpLOxs{zHz!BJ?tJbODa&o_kpuoxizFuiE$b^Qud`T?z=Rqc=M< zeY>~t!zf95ENg=`%b5q|Z7$K858(FW{qOgbQa6u~49#P81LH!^cjK z^*3)}ki+kl9jAy?UJ52gwP$;(RT&dT278aj;j?niY;SLzij362xJbakl_VQ!pJ+pz zx33`x_Ztn!s^ewf%0i%*XwsT7_OO6Av2v^9ziM+OVRH2|~V98*3xKOBAhx`bi8HwKhy!Ra)^hQX*p=b@HZAmclcJw)9Eu6%Z~8I8o=y562n%2f}of#>!#*flB4hq zwvjr%du~F~L;|v)4FrsMK$>9Cc~kKI?Rp0|U*N~4oDJ_~8IH|f0RogFvvllc7v|9S z-A)txSoq{p&!{nxq%$!xQ4)C;?-5jCJ|bdchlS;<5Yi?G9`pF0oeX0YwA_@bZQ8z_iL!zy1(FrB^W)xrVqOKpbl4 zF4oIU;W&2)zOcocU-hjI22L_-?Uj&c&~|Bgh;~SRJ>pR8rS1e_+!$QD8KJ_O{*59OQC2 zj?-DDh4+wS+N?W_#|Q1P(0Sop^*du#B7rq=1qFkCh&6GU?xIfy_45rnp9o=O)o+4O z>ouPm7Bx^h+|eBFtIodu1s~w}jAGKumFZ8=kK#|EMaHiYBnLSM_Xv0sGZ1tHmaPK& zYV|H+RPvIP*AL0Z&UdXRrXu7Wk~{j}*KDYuLTm~n=vP}Lu5j<{>0*df7Xj}iG~WDGo48;Nal zZKPX^I11PeSh6g%lKiHX^jT3mjD6K#kfGHbMfS3eGm^7CHy~C<_4&&>3%|RCWeEEm z#O^HU@d-Jtc%FSw@;Y+E`hINXEQA}{MhI|iT)63lVvw%JqKd0_FZ9NJ*t%+%T$U4o zvW%1tYUs1*Xy{`=ZjA{@>Ms)8|u$pZ`Ak$8KVsHQKr#7WsP*T4!_Hj7;mX2$N0Q;2{k1bF8F6)+qr~ zM%SrQhQ*>2OTWNVxl%Ktl@BVz)0bFqw7rkQ&VqGMmnCFGNWcL)X-B7+$h|=RZASHb ziKahZ6HWdZg7wylGb!>pbUfMKRD&qo-Mqwe`WjE4rbQ3q3GUzsprhAbrALQzZS50p zyll0>G?;${&Eh1|Kv4YWGn=8~+H2x5i)??MH+SV{8p<`B=S!C>N zvO8Jh67{)7)ea;+>k-G4%U4^EN`7PmUGp-7#)DRY9eB#Dkzrwbe*)($02Gr&{~a{9 z_3ze0L?ybyjfk0r>AX!k6uOZpc6_IZm7n9%I&E->2)m-_wRCa)SDsolepAd2Mu1)S ztbW(nJ*~Yv z$rQlj_Hcp*Ab)_3xJ)qGOZ&WeF~f)SC8c224`F<(i#(DK-Zd|=>lYuJR+m|bH%yL& zS5j~De61(V_;?vj+ClJJj)F3$m2iqy1=o%%1O6Gf&&*)FtA!bcEA5n*1juq{F@~O1 z#AWFv@T*}0IF7SFm+I1xa&m_%b!~ozwvj6Es_e#ht=7v!1^5MD=Mw_d?$0DJ@E9_` zU|*MM#`=XmVPw5YF2CRS$v^*eb`D@O#ez*3a=Jiw?q4Dn*2>K_kvX$G69iHAd{I=E zhk#35LfZZ!4~m34VrN3+jk4+>;Hj`1nFgc@TYcA!F)(ybZN zkRi6+w8FX!9s8=y%>!p?;04Yuu<$>W|2cMDREKK0r}e17G>IdFRGsVwo;rAJE9Gxt zmohx_cA(4Y_Gl6`!r++ZpBbF&tl^vsaWhwTQ&D!DEvw!6JMcwHRyG`_XIL7If;_oi bV<`A7G(w&j?v4)$LIh3>fA9a Date: Tue, 10 Dec 2024 14:14:31 +0000 Subject: [PATCH 15/17] [pre-commit.ci] pre-commit autoupdate (#2612) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b7004b93d..05934f3059 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.8.1" + rev: "v0.8.2" hooks: - id: ruff args: [--fix] From 5353ab03fadbdc0afcb2bc5d9af6c02823ffa6a8 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 10 Dec 2024 14:21:33 +0000 Subject: [PATCH 16/17] switch back to Python 3.12 for conda lock file creation due to mamba<2 pin (#2606) --- .github/workflows/create-condalock-file.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 97501f657c..6a896a1d43 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -27,7 +27,7 @@ jobs: with: auto-update-conda: true activate-environment: esmvaltool-fromlock - python-version: "3.13" + python-version: "3.12" # switch to 3.13 when mamba>2 available miniforge-version: "latest" use-mamba: true - name: Update and show conda config From e09e39640331fb9a72d2f281861565dda5eea03c Mon Sep 17 00:00:00 2001 From: Karen Garcia Perdomo <85649962+Karen-A-Garcia@users.noreply.github.com> Date: Tue, 10 Dec 2024 06:24:02 -0800 Subject: [PATCH 17/17] Monotonicity fixes for Fgoals (#2603) Co-authored-by: Karen Garcia Perdomo --- esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py | 76 +++++++++++++++++++ .../cmor/_fixes/cmip6/test_cesm2.py | 2 +- .../cmor/_fixes/cmip6/test_fgoals_g3.py | 72 +++++++++++++++++- 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py b/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py index 591fa54b86..2d5206f8c8 100644 --- a/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py +++ b/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py @@ -2,6 +2,7 @@ import dask.array as da import iris +import numpy as np from ..common import OceanFixGrid from ..fix import Fix @@ -84,3 +85,78 @@ def fix_metadata(self, cubes): iris.util.promote_aux_coord_to_dim_coord(cube, "longitude") return super().fix_metadata(cubes) + + +class Tas(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """Fix time coordinates. + + Parameters + ---------- + cubes : iris.cube.CubeList + Cubes to fix + + Returns + ------- + iris.cube.CubeList + """ + new_list = iris.cube.CubeList() + for cube in cubes: + try: + old_time = cube.coord("time") + except iris.exceptions.CoordinateNotFoundError: + new_list.append(cube) + else: + if old_time.is_monotonic(): + new_list.append(cube) + else: + time_units = old_time.units + time_data = old_time.points + + # erase erroneously copy-pasted points + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + while len(idx_neg) > 0: + time_data = np.delete(time_data, idx_neg[0] + 1) + time_diff = np.diff(time_data) + idx_neg = np.where(time_diff <= 0.0)[0] + + # create the new time coord + new_time = iris.coords.DimCoord( + time_data, + standard_name="time", + var_name="time", + units=time_units, + ) + + # create a new cube with the right shape + dims = ( + time_data.shape[0], + cube.coord("latitude").shape[0], + cube.coord("longitude").shape[0], + ) + data = cube.data + new_data = np.ma.append( + data[: dims[0] - 1, :, :], data[-1, :, :] + ) + new_data = new_data.reshape(dims) + + tmp_cube = iris.cube.Cube( + new_data, + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=[ + (new_time, 0), + (cube.coord("latitude"), 1), + (cube.coord("longitude"), 2), + ], + ) + + new_list.append(tmp_cube) + return new_list diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 24df5db059..0bccf89186 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -507,7 +507,7 @@ def test_pr_fix_metadata(pr_cubes): out_cubes = fix.fix_metadata(pr_cubes) for cube in out_cubes: - if cube.var_name == "tas": + if cube.var_name == "pr": assert cube.coord("time").is_monotonic() diff --git a/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py b/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py index eb16a4d2ba..0621c56221 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py +++ b/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py @@ -4,8 +4,10 @@ import iris import numpy as np +import pandas as pd +import pytest -from esmvalcore.cmor._fixes.cmip6.fgoals_g3 import Mrsos, Siconc, Tos +from esmvalcore.cmor._fixes.cmip6.fgoals_g3 import Mrsos, Siconc, Tas, Tos from esmvalcore.cmor._fixes.common import OceanFixGrid from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -163,3 +165,71 @@ def test_mrsos_fix_metadata_2(mock_base_fix_metadata): [[0.5, 1.5], [1.5, 2.5], [2.5, 3.5]], ) mock_base_fix_metadata.assert_called_once_with(fix, cubes) + + +@pytest.fixture +def tas_cubes(): + correct_time_coord = iris.coords.DimCoord( + points=[1.0, 2.0, 3.0, 4.0, 5.0], + var_name="time", + standard_name="time", + units="days since 1850-01-01", + ) + + lat_coord = iris.coords.DimCoord( + [0.0], var_name="lat", standard_name="latitude" + ) + + lon_coord = iris.coords.DimCoord( + [0.0], var_name="lon", standard_name="longitude" + ) + + correct_coord_specs = [ + (correct_time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + + correct_tas_cube = iris.cube.Cube( + np.ones((5, 1, 1)), + var_name="tas", + units="K", + dim_coords_and_dims=correct_coord_specs, + ) + + scalar_cube = iris.cube.Cube(0.0, var_name="ps") + + return iris.cube.CubeList([correct_tas_cube, scalar_cube]) + + +def test_get_tas_fix(): + """Test tas fix.""" + fix = Fix.get_fixes("CMIP6", "FGOALS-g3", "day", "tas") + assert fix == [Tas(None), GenericFix(None)] + + +def test_tas_fix_metadata(tas_cubes): + """Test metadata fix.""" + vardef = get_var_info("CMIP6", "day", "tas") + fix = Tas(vardef) + + out_cubes = fix.fix_metadata(tas_cubes) + assert out_cubes[0].var_name == "tas" + coord = out_cubes[0].coord("time") + assert pd.Series(coord.points).is_monotonic_increasing + + # de-monotonize time points + for cube in tas_cubes: + if cube.var_name == "tas": + time = cube.coord("time") + points = np.array(time.points) + points[-1] = points[0] + dims = cube.coord_dims(time) + cube.remove_coord(time) + time = iris.coords.AuxCoord.from_coord(time) + cube.add_aux_coord(time.copy(points), dims) + + out_cubes = fix.fix_metadata(tas_cubes) + for cube in out_cubes: + if cube.var_name == "tas": + assert cube.coord("time").is_monotonic()