From bb9f413b145339198ccaa2df5823d1d14160aaef Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 19 Sep 2023 17:08:15 +0200 Subject: [PATCH 01/72] Fix sorting of ensemble members in `esmvalcore.dataset.datasets_to_recipe` (#2095) --- esmvalcore/_recipe/from_datasets.py | 1 + tests/unit/recipe/test_from_datasets.py | 35 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index 145c2e95d0..c54931806f 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -171,6 +171,7 @@ def grouper(facets): return tuple((k, facets[k]) for k in sorted(facets) if k != 'ensemble') result = [] + dataset_facets = sorted(dataset_facets, key=grouper) for group_facets, group in itertools.groupby(dataset_facets, key=grouper): ensembles = [f['ensemble'] for f in group if 'ensemble' in f] if not ensembles: diff --git a/tests/unit/recipe/test_from_datasets.py b/tests/unit/recipe/test_from_datasets.py index 62f83eae34..137f55f0c1 100644 --- a/tests/unit/recipe/test_from_datasets.py +++ b/tests/unit/recipe/test_from_datasets.py @@ -4,6 +4,7 @@ import yaml from esmvalcore._recipe.from_datasets import ( + _group_ensemble_members, _group_ensemble_names, _group_identical_facets, _move_one_level_up, @@ -211,6 +212,40 @@ def test_group_identical_facets(): assert result == expected +def test_group_ensemble_members(): + datasets = [ + Dataset( + dataset='dataset1', + ensemble='r1i1p1f1', + grid='gn', + ), + Dataset( + dataset='dataset1', + ensemble='r1i1p1f1', + grid='gr1', + ), + Dataset( + dataset='dataset1', + ensemble='r2i1p1f1', + grid='gn', + ), + ] + result = _group_ensemble_members(ds.facets for ds in datasets) + print(result) + assert result == [ + { + 'dataset': 'dataset1', + 'ensemble': 'r(1:2)i1p1f1', + 'grid': 'gn', + }, + { + 'dataset': 'dataset1', + 'ensemble': 'r1i1p1f1', + 'grid': 'gr1', + }, + ] + + def test_group_ensembles_cmip5(): ensembles = [ "r1i1p1", From 09e20d41e4590f5b146672727964ab1d0849224b Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:00:29 +0200 Subject: [PATCH 02/72] Call coord.core_bounds() instead of coord.bounds in `check.py` (#2146) --- esmvalcore/cmor/check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 29f0f74e7e..4eb057730b 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -773,8 +773,8 @@ def _check_coord_points(self, coord_info, coord, var_name): else: new_lons = coord.core_points().copy() new_lons = self._set_range_in_0_360(new_lons) - if coord.bounds is not None: - new_bounds = coord.bounds.copy() + if coord.core_bounds() is not None: + new_bounds = coord.core_bounds().copy() new_bounds = self._set_range_in_0_360(new_bounds) else: new_bounds = None From 55313b5a3eb7c55e71f4ee4fc7e34697239f4329 Mon Sep 17 00:00:00 2001 From: Klaus Zimmermann Date: Tue, 26 Sep 2023 16:47:09 +0200 Subject: [PATCH 03/72] Ensure compatible zstandard and zstd versions for .conda support (#2204) --- .readthedocs.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c95c86a3fc..2e4d195c1c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,9 +14,11 @@ build: jobs: pre_create_environment: # update mamba just in case - - mamba update --yes --quiet --name=base mamba + - mamba update --yes --quiet --name=base mamba 'zstd=1.5.2' - mamba --version + - mamba list --name=base post_create_environment: + - conda run -n ${CONDA_DEFAULT_ENV} mamba list # use conda run executable wrapper to have all env variables - conda run -n ${CONDA_DEFAULT_ENV} mamba --version - conda run -n ${CONDA_DEFAULT_ENV} pip install . --no-deps From 60e0d2bcb9c9190dd069dd2b12a740f7f8914276 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:34:52 +0100 Subject: [PATCH 04/72] [Condalock] Update Linux condalock file (#2196) Co-authored-by: valeriupredoi Co-authored-by: Valeriu Predoi --- conda-linux-64.lock | 188 ++++++++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index 7a302cde45..a5ec825a8a 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -9,24 +9,22 @@ https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed3 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2#4d59c254e01d9cde7957100457e2d5fb https://conda.anaconda.org/conda-forge/noarch/font-ttf-ubuntu-0.83-hab24e00_0.tar.bz2#19410c3df09dfb12d1206132a1d357c5 https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_16.conda#7ca122655873935e02c91279c5b03c8c -https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.39-hcc3a1bd_1.conda#737be0d34c22d24432049ab7a3214de4 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-12.2.0-h3b97bd3_19.tar.bz2#199a7292b1d3535376ecf7670c231d1f -https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_0.conda#aa3ee989b0eba634e47197adbaa84fdd -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-12.2.0-h3b97bd3_19.tar.bz2#277d373b57791ee71cafc3c5bfcf0641 -https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_0.conda#47d33bfb38632133412c20d1dee8eeae +https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda#7aca3059a1729aa76c597603f10b0dd3 +https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-12.3.0-h8bca6fd_2.conda#ed613582de7b8569fdc53ca141be176a +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-devel_linux-64-12.3.0-h8bca6fd_2.conda#7268a17e56eb099d1b8869bbbf46de4c +https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h7e041cc_2.conda#9172c297304f2a20134fc56c97fbe229 https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda#d8d7293c5b37f39b2ac32940621c6592 -https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-3_cp311.conda#c2e2630ddb68cf52eec74dc7dfab20b5 +https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.11-4_cp311.conda#d786502c97404c94d7d58d258a445a65 https://conda.anaconda.org/conda-forge/noarch/tzdata-2023c-h71feb2d_0.conda#939e3e74d8be4dac89ce83b20de2492a https://conda.anaconda.org/conda-forge/noarch/fonts-conda-forge-1-0.tar.bz2#f766549260d6815b0c52253f1fb1bb29 -https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_0.conda#bdaebefa4f6c436741187c8a304681a5 -https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_0.conda#33c2353e425610c69ad58a8e29694724 +https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_2.conda#e2042154faafe61969556f28bade94b9 https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.12-he073ed8_16.conda#071ea8dceff4d30ac511f4a2f8437cd1 https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d -https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.39-he00db2b_1.conda#3d726e8b51a1f5bfd66892a2b7d9db2d +https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.40-hf600244_0.conda#33084421a8c0af6aef1b439707f7662a https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2#fee5683a3f04bd15cbd8318b096a27ab -https://conda.anaconda.org/conda-forge/linux-64/binutils-2.39-hdd6e379_1.conda#1276c18b0a562739185dbf5bd14b57b2 -https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.39-h5fc0e48_13.conda#7f25a524665e4e2f8a5f86522f8d0e31 -https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_0.conda#3934dca6107fad668d036a4cafca1015 +https://conda.anaconda.org/conda-forge/linux-64/binutils-2.40-hdd6e379_0.conda#ccc940fddbc3fcd3d79cd4c654c4b5c4 +https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.40-hbdbef99_2.conda#adfebae9fdc63a598495dfe3b006973a +https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_2.conda#c28003b0be0494f9a7664389146716ff https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.0-hd590300_0.conda#71b89db63b5b504e7afc8ad901172e1e https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54 https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.19.1-hd590300_0.conda#e8c18d865be43e2fb3f7a145b6adf1f5 @@ -42,7 +40,7 @@ https://conda.anaconda.org/conda-forge/linux-64/icu-73.2-h59595ed_0.conda#cc47e1 https://conda.anaconda.org/conda-forge/linux-64/json-c-0.17-h7ab15ed_0.conda#9961b1f100c3b6852bd97c9233d06979 https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz2#30186d27e2c9fa62b45fb1476b7200e3 https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f -https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230125.3-cxx17_h59595ed_0.conda#d1db1b8be7c3a8983dcbbbfe4f0765de +https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230802.1-cxx17_h59595ed_0.conda#2785ddf4cb0e7e743477991d64353947 https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda#0f683578378cddb223e7fd24f785ab2a https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_0.conda#e805cbec4c29feb22e019245f7e47b6c https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 @@ -50,12 +48,12 @@ https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.18-h0b41bf4_0.conda https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 +https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_2.conda#78fdab09d9138851dde2b5fe2a11019e https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d -https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-h0b41bf4_0.conda#1edd9e67bdb90d78cea97733ff6b54e6 +https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-hd590300_1.conda#323e90742f0f48fc22bea908735f55e6 https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2#39b1328babf85c7c3a61636d9cd50206 https://conda.anaconda.org/conda-forge/linux-64/libnuma-2.0.16-h0b41bf4_1.conda#28bfe2cb11357ccc5be21101a6b7ce86 -https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 -https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.2.0-h46fd767_19.tar.bz2#80d0e00150401e9c06a055f36e8e73f2 +https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.3.0-h0f45ef3_2.conda#4655db64eca78a6fcc4fb654fc1f8d57 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.18-h36c2ea0_1.tar.bz2#c3788462a6fbddafdb413a9f9053e58d https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda#f204c8ba400ec475452737094fb81d52 https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar.bz2#ede4266dc02e875fe1ea77b25dd43747 @@ -66,7 +64,7 @@ https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318 https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2#bb14fcb13341b81d5eb386423b9d2bac https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda#681105bccc2a3f7f1a837d47d39c9179 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 -https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.2-hd590300_0.conda#e5ac5227582d6c83ccf247288c0eb095 +https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.3-hd590300_0.conda#7bb88ce04c8deb9f7d763ae04a1da72f https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2#660e72c82f2e75a6b3fe6a6e75c79f19 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 https://conda.anaconda.org/conda-forge/linux-64/rdma-core-28.9-h59595ed_1.conda#aeffb7c06b5f65e55e6c637408dc4100 @@ -83,61 +81,60 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007 https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.2-hd590300_0.conda#f08fb5c89edfc4aadee1c81d4cfb1fa1 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.2-hc309b26_0.conda#93b55df578f8c552e9480bae939daf36 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.2-hc309b26_1.conda#72b0dc52522915a4665a55707cc34af0 https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h4d4d85c_2.conda#9ca99452635fe03eb5fa937f5ae604b0 https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h4d4d85c_1.conda#eba092fc6de212a01de0065f38fe8bbb https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h4d4d85c_1.conda#30f9df85ce23cd14faa9a4dfa50cca2b https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda#8b9b5aca60558d02ddaa09d599e55920 -https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.2.0-hcc96c02_19.tar.bz2#bb48ea333c8e6dcc159a1575f04d869e +https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.3.0-he2b93b0_2.conda#2f4d8677dc7dd87f93e9abfb2ce86808 https://conda.anaconda.org/conda-forge/linux-64/glog-0.6.0-h6f12383_0.tar.bz2#b31f3565cb84435407594e548a2fb7b2 https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h501b40f_6.conda#c3e9338e15d90106f467377017352b97 -https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-18_linux64_openblas.conda#bcddbb497582ece559465b9cd11042e7 https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_0.conda#43017394a280a42b48d11d2a6e169901 https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_0.conda#8e3e1cb77c4b355a3776bdfb74095bed https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d +https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_2.conda#e75a75a6eaf6f318dae2631158c46575 https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.52.0-h61bc06f_0.conda#613955a50485812985c059e7b269f42e https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.39-h753d276_0.conda#e1c890aebdebbfbf87e2c917187b4416 -https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.23.3-hd1fb520_1.conda#78c10e8637a6f8d377f9989327d0267d +https://conda.anaconda.org/conda-forge/linux-64/libprotobuf-4.23.4-hf27288f_6.conda#f28b3651e20e63f7da58798880061089 https://conda.anaconda.org/conda-forge/linux-64/librttopo-1.1.0-hb58d41b_14.conda#264f9a3a4ea52c8f4d3e8ae1213a3335 https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.43.0-h2797004_0.conda#903fa782a9067d5934210df6d79220f6 https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.11.0-h0841786_0.conda#1f5a58e686b13bcfde88b93f547d23fe https://conda.anaconda.org/conda-forge/linux-64/libxcb-1.15-h0b41bf4_0.conda#33277193f5b92bad9fdd230eb700929c https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.11.5-h232c23b_1.conda#f3858448893839820d4bcfb14ad3ecdf -https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_2.conda#a83ad320127e83ae8a86b3db8dfeec77 +https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_3.conda#ac79812548e7e8cf61f7b0abdef01d3b https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2#69e2c796349cd9b273890bee0febfe1b https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 -https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.49-h06160fa_0.conda#1d78349eb26366ecc034a4afe70a8534 +https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.51-h06160fa_0.conda#cd63086544e897be1006fc2d88ed1fe8 https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2#5b8c42eb62e9fc961af70bdd6a26e168 -https://conda.anaconda.org/conda-forge/linux-64/ucx-1.14.1-h64cca9d_4.conda#bbbd3de252bf44be87dc83dbf8a5b653 +https://conda.anaconda.org/conda-forge/linux-64/ucx-1.14.1-h64cca9d_5.conda#39aa3b356d10d7e5add0c540945a0944 +https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_1.conda#85552d64cb49f12781668779efc738ec https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda#93ee23f12bc2e684548181256edd2cf6 https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.4-h9c3ff4c_1.tar.bz2#21743a8d2ea0c8cfbbf8fe489b0347df https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda#68c34ec6149623be41a1933ab996a209 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda#04b88013080254850d6c01ed54810589 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.32-h019f825_2.conda#cfb2918b63e33374a5f19330bcd21c6a +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.32-h1a03231_3.conda#9e2dd8e0e39417f2de68ac1018cdc809 https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda#009521b7ed97cca25f8f997f9e745976 https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h2c5509c_4.conda#417a9d724dc4b651f4a711d3aa3694e3 https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_0.conda#aeafb07a327e3f14a796bf081ea07472 -https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-hca18f0e_1.conda#e1232042de76d24539a436d37597eb06 -https://conda.anaconda.org/conda-forge/linux-64/gcc-12.2.0-h26027b1_13.conda#ec93d13e0fe8514f65842120dbae1b16 -https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.2.0-h4798a0e_13.conda#1d009211292e0d869a31f3bc5b4ee78b -https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.2.0-h55be85b_19.tar.bz2#143d770a2a2911cd84b98286db0e6a40 -https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.2.0-hcc96c02_19.tar.bz2#698aae34e4f5e0ea8eac0d529c8f20b6 +https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda#9ae35c3d96db2c94ce0cef86efdfa2cb +https://conda.anaconda.org/conda-forge/linux-64/gcc-12.3.0-h8d2909c_2.conda#e2f2f81f367e14ca1f77a870bda2fe59 +https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.3.0-h76fc315_2.conda#11517e7b5c910c5b5d6985c0c7eb7f50 +https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-12.3.0-hfcedea8_2.conda#09d48cadff6669068c3bf7ae7dc8ea4a +https://conda.anaconda.org/conda-forge/linux-64/gxx_impl_linux-64-12.3.0-he2b93b0_2.conda#f89b9916afc36fc5562fbfc11330a8a2 https://conda.anaconda.org/conda-forge/linux-64/krb5-1.21.2-h659d440_0.conda#cd95826dbd331ed1be26bdf401432844 -https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.6.2-h039dbb9_1.conda#29cf970521d30d113f3425b84cb250f6 -https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-18_linux64_openblas.conda#93dd9ab275ad888ed8113953769af78c +https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.2-h039dbb9_0.conda#611d6c83d1130ea60c916531adfb11db https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.0-hebfc3b9_0.conda#e618003da3547216310088478e475945 -https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.56.2-h3905398_1.conda#0b01e6ff8002994bd4ddbffcdbec7856 -https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-18_linux64_openblas.conda#a1244707531e5b143c420c70573c8ec5 +https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.57.0-ha4d0f93_1.conda#56ce4bcc0e1cd0b4c3d7149010410e9a +https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.19.0-h8fd135c_0.conda#d5d149effb0fe13805b68ac2afd242b1 https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.1-h8b53f26_1.conda#5b09e13d732dda1a2bc9adc711164f4d https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h0054252_1.conda#f27960e8873abb5476e96ef33bdbdccd https://conda.anaconda.org/conda-forge/linux-64/nss-3.92-h1d7d5a4_0.conda#22c89a3d87828fe925b310b9cdf0f574 -https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h385abfd_1.conda#2cd5aac7ef1b4c6ac51bf521251a89b3 +https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h52d3b3c_2.conda#6e1931d3d8512593f606aa08d9bd5192 https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.1.3-h32600fe_0.conda#8287aeb8462e2d4b235eff788e75919d https://conda.anaconda.org/conda-forge/linux-64/python-3.11.5-hab00c5b_0_cpython.conda#f0288cb82594b1cbc71111d1cd3c5422 https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.43.0-h2c6b66d_0.conda#713f9eac95d051abe14c3774376854fe -https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-hc3e0081_0.tar.bz2#d4c341e0379c31e9e781d4f204726867 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.6-h8ee46fc_0.conda#7590b76c3d11d21caa44f3fc38ac584a https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.13-pyhd8ed1ab_0.conda#06006184e203b61d3525f90de394471e https://conda.anaconda.org/conda-forge/linux-64/antlr-python-runtime-4.7.2-py311h38be061_1003.tar.bz2#0ab8f8f0cae99343907fe68cda11baea @@ -164,22 +161,22 @@ https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.b https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2#961b3a227b437d82ad7054484cfa71b2 https://conda.anaconda.org/conda-forge/noarch/dill-0.3.7-pyhd8ed1ab_0.conda#5e4f3466526c52bc9af2d2353a1460bd https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.7-pyhd8ed1ab_0.conda#12d8aae6994f342618443a8f05c652a0 -https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_0.conda#207175b7d514d42f977ec505800d6824 +https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_1.conda#ff1b48b5b802afe76597197113b8e64d https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b https://conda.anaconda.org/conda-forge/noarch/entrypoints-0.4-pyhd8ed1ab_0.tar.bz2#3cf04868fee0a029769bd41f4b2fbf2d https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.3-pyhd8ed1ab_0.conda#e6518222753f519e911e83136d2158d9 https://conda.anaconda.org/conda-forge/noarch/execnet-2.0.2-pyhd8ed1ab_0.conda#67de0d8241e1060a479e3c37793e26f9 https://conda.anaconda.org/conda-forge/noarch/executing-1.2.0-pyhd8ed1ab_0.tar.bz2#4c1bc140e2be5c8ba6e3acab99e25c50 -https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.3-pyhd8ed1ab_0.conda#3104cf0ab9fb9de393051bf92b10dbe9 +https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.4-pyhd8ed1ab_0.conda#5173d4b8267a0699a43d73231e0b6596 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda#0f69b688f52ff6da70bccb7ff7001d1d -https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.0-pyh1a96a4e_0.conda#b4a3c7bb3f45d47e085764ff096fa259 +https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.1-pyh1a96a4e_0.conda#d69753ff6ee3c84a6638921dd95db662 https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h6b639ba_2.conda#ee8220db21db8094998005990418fe5b https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 -https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.2.0-h8acd90e_13.conda#ca62fa3fcd15a7d92194ee2ff7c9c54b -https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-12.2.0-h307d370_13.conda#d4a4dd80f5a470407eed32cb2932174a +https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.3.0-h499e0f7_2.conda#0558a8c44eb7a18e6682bd3a8ae6dcab +https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-12.3.0-h7fe76b4_2.conda#3a749210487c0358b6f135a648cbbf60 https://conda.anaconda.org/conda-forge/linux-64/gts-0.7.6-h977cf35_4.conda#4d8df0b0db060d33c9a702ada998a8fe -https://conda.anaconda.org/conda-forge/linux-64/gxx-12.2.0-h26027b1_13.conda#de605ff437f3fdc010f1b529642339f1 -https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-12.2.0-hb41e900_13.conda#cf45659115653fc80623116ccd0a6ae2 +https://conda.anaconda.org/conda-forge/linux-64/gxx-12.3.0-h8d2909c_2.conda#673bac341be6b90ef9e8abae7e52ca46 +https://conda.anaconda.org/conda-forge/linux-64/gxx_linux-64-12.3.0-h8a814eb_2.conda#f517b1525e9783849bd56a5dc45a9960 https://conda.anaconda.org/conda-forge/linux-64/humanfriendly-10.0-py311h38be061_4.tar.bz2#5c4f38a9e482f00a7bf23fe479c8ca29 https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#34272b248891bddccc64479f9a7fffed https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 @@ -188,7 +185,8 @@ https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.ta https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py311h9547e67_0.conda#f53903649188b99e6b44c560c69f5b23 https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h2582759_0.conda#07745544b144855ed4514a4cf0aadd74 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-haa2dc70_1.conda#980d8aca0bc23ca73fa8caa3e7c84c28 -https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.2.1-hca28451_0.conda#96aec6156d58591f5a4e67056521ce1b +https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-18_linux64_openblas.conda#bcddbb497582ece559465b9cd11042e7 +https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.3.0-hca28451_0.conda#4ab41bee09a2d2e08de5f09d6f1eef62 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 https://conda.anaconda.org/conda-forge/linux-64/libpq-15.4-hfc447b1_0.conda#b9ce311e7aba8b5fc3122254f0a6e97e https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.1-hbf2b3c1_0.conda#4963f3f12db45a576f2b8fbe9a0b8569 @@ -203,14 +201,13 @@ https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda#376 https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda#4eccaeba205f0aed9ac3a9ea58568ca3 https://conda.anaconda.org/conda-forge/noarch/networkx-3.1-pyhd8ed1ab_0.conda#254f787d5068bc89f578bf63893ce8b4 -https://conda.anaconda.org/conda-forge/linux-64/numpy-1.25.2-py311h64a7726_0.conda#71fd6f1734a0fa64d8f852ae7156ec45 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda#5ce6a42505c6e9e6151c54c3ec8d68ea https://conda.anaconda.org/conda-forge/noarch/packaging-23.1-pyhd8ed1ab_0.conda#91cda59e66e1e4afe9476f8ef98f5c30 https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9 https://conda.anaconda.org/conda-forge/noarch/parso-0.8.3-pyhd8ed1ab_0.tar.bz2#17a565a0c3899244e938cdf417e7b094 https://conda.anaconda.org/conda-forge/noarch/pathspec-0.11.2-pyhd8ed1ab_0.conda#e41debb259e68490e3ab81e46b639ab6 https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-py_1003.tar.bz2#415f0ebb6198cc2801c73438a9fb5761 -https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_0.tar.bz2#89e3c7cdde7d3aaa2aee933b604dd07f +https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_1.conda#405678b942f2481cecdb3e010f4925d9 https://conda.anaconda.org/conda-forge/noarch/pluggy-1.3.0-pyhd8ed1ab_0.conda#2390bd10bed1f3fdc7a537fb5a447d8d https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h2582759_0.conda#a90f8e278c1cd7064b2713e6b7db87e6 https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd3deb0d_0.tar.bz2#359eeb6536da0e687af562ed265ec263 @@ -220,7 +217,7 @@ https://conda.anaconda.org/conda-forge/noarch/pycodestyle-2.9.1-pyhd8ed1ab_0.tar https://conda.anaconda.org/conda-forge/noarch/pycparser-2.21-pyhd8ed1ab_0.tar.bz2#076becd9e05608f8dc72757d5f3a91ff https://conda.anaconda.org/conda-forge/noarch/pyflakes-2.5.0-pyhd8ed1ab_0.tar.bz2#1b3bef4313288ae8d35b1dfba4cd84a3 https://conda.anaconda.org/conda-forge/noarch/pygments-2.16.1-pyhd8ed1ab_0.conda#40e5cb18165466773619e5c963f00a7b -https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.0.9-pyhd8ed1ab_0.tar.bz2#e8fbc1b54b25f4b08281467bc13b70cc +https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.1.1-pyhd8ed1ab_0.conda#176f7d56f0cfe9008bdf1bccd7de02fb https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#92a889dc236a5197612bc85bee6d7174 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.18.0-pyhd8ed1ab_0.conda#3be9466311564f80f8056c0851fc5bb7 @@ -229,10 +226,10 @@ https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.3.0-py311h459d7e https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3.post1-pyhd8ed1ab_0.conda#c93346b446cd08c169d843ae5fc0da97 https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_0.conda#30eaaf31141e785a445bf1ede6235fe3 https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h75c88c4_0.conda#af6d43afe0d179ac83b7e0c16b2caaad -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.2-py311h46250e7_0.conda#0068f069d641ebeb5a9d8f3422f90442 +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.3-py311h46250e7_0.conda#da1b2b57ac17853cfeb4197d0595db45 https://conda.anaconda.org/conda-forge/noarch/semver-3.0.1-pyhd8ed1ab_0.conda#ed90854ae56fb6edae1f13b4663b21b0 https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df -https://conda.anaconda.org/conda-forge/noarch/setuptools-68.1.2-pyhd8ed1ab_0.conda#4fe12573bf499ff85a0a364e00cc5c53 +https://conda.anaconda.org/conda-forge/noarch/setuptools-68.2.2-pyhd8ed1ab_0.conda#fc2166155db840c634a1291a5c35a709 https://conda.anaconda.org/conda-forge/noarch/six-1.16.0-pyh6c4a22f_0.tar.bz2#e5f25f8dbc060e9a8d912e432202afc2 https://conda.anaconda.org/conda-forge/noarch/smmap-3.0.5-pyh44b312d_0.tar.bz2#3a8dc70789709aa315325d5df06fb7e4 https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-2.2.0-pyhd8ed1ab_0.tar.bz2#4d22a9315e78c6827f806065957d566e @@ -240,20 +237,20 @@ https://conda.anaconda.org/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_ https://conda.anaconda.org/conda-forge/noarch/soupsieve-2.5-pyhd8ed1ab_1.conda#3f144b2c34f8cb5a9abd9ed23a39c561 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_0.conda#da1d979339e2714c30a8e806a33ec087 https://conda.anaconda.org/conda-forge/noarch/sqlparse-0.4.4-pyhd8ed1ab_0.conda#2e2f31b3b1c866c29636377e14f8c4c6 -https://conda.anaconda.org/conda-forge/noarch/tblib-1.7.0-pyhd8ed1ab_0.tar.bz2#3d4afc31302aa7be471feb6be048ed76 +https://conda.anaconda.org/conda-forge/noarch/tblib-2.0.0-pyhd8ed1ab_0.conda#f5580336fe091d46f9a2ea97da044550 https://conda.anaconda.org/conda-forge/noarch/termcolor-2.3.0-pyhd8ed1ab_0.conda#440d508f025b1692168caaf436504af3 https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f832c45a477c78bebd107098db465095 https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.12.1-pyha770c72_0.conda#62f5b331c53d73e2f6c4c130b53518a0 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_0.conda#7d9a31416c18704f55946ff7cf8da5dc -https://conda.anaconda.org/conda-forge/noarch/traitlets-5.9.0-pyhd8ed1ab_0.conda#d0b4f5c87cd35ac3fb3d47b223263a64 +https://conda.anaconda.org/conda-forge/noarch/traitlets-5.10.0-pyhd8ed1ab_0.conda#efd3f63a93621367d4fa6e274c511696 https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.11-pyhd8ed1ab_0.conda#22776dce28e8ba933e5cbcf20b62c583 https://conda.anaconda.org/conda-forge/noarch/types-urllib3-1.26.25.14-pyhd8ed1ab_0.conda#06118f39abab2ab953276a50b2775509 -https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.7.1-pyha770c72_0.conda#c39d6a09fe819de4951c2642629d9115 +https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.8.0-pyha770c72_0.conda#5b1be40a26d10a06f6d4f1f9e19fa0c7 https://conda.anaconda.org/conda-forge/linux-64/ujson-5.7.0-py311hcafe171_0.conda#ec3960b6d13bb60aad9c67f42a801720 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-py_0.tar.bz2#1447ead40f2a01733a9c8dfc32988375 -https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2#3563be4c5611a44210d9ba0c16113136 +https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 https://conda.anaconda.org/conda-forge/noarch/webob-1.8.7-pyhd8ed1ab_0.tar.bz2#a8192f3585f341ea66c60c189580ac67 https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.2-pyhd8ed1ab_0.conda#1ccd092478b3e0ee10d7a891adbf8a4f https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h2582759_0.conda#15565d8602a78c6a994e4d9fcb391920 @@ -261,7 +258,7 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.co https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda#ed67c36f215b310412b2af935bf3e530 https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.7.0-pyhd8ed1ab_0.conda#aacae3c0eaba0204dc6c5497c93c7992 https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30c2c15b82aacb07f9c09e28ff2275 -https://conda.anaconda.org/conda-forge/noarch/zipp-3.16.2-pyhd8ed1ab_0.conda#2da0451b54c4563c32490cb1b7cf68a1 +https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda#2e4d6bc0b14e10f895fc6791a7d9b26a https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.4-pyhd8ed1ab_0.conda#46a2e6e3dfa718ce3492018d5a110dd6 https://conda.anaconda.org/conda-forge/noarch/asgiref-3.7.2-pyhd8ed1ab_0.conda#596932155bf88bb6837141550cb721b0 https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.6-py311h38be061_0.conda#28b1d4a493fb6acd24cc299d77fed871 @@ -276,10 +273,8 @@ https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-h0c91306_1017.conda https://conda.anaconda.org/conda-forge/noarch/cattrs-23.1.2-pyhd8ed1ab_0.conda#e554f60477143949704bf470f66a81e7 https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda#9025d0786dbbe4bc91fd8e85502decce https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda#797554b8b7603011e8677884381fbcc5 -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h4c7f6c3_1.tar.bz2#c7e54004ffd03f8db0a58ab949f2a00b https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.0-py311h9547e67_0.conda#daf3f23397ab2265d0cdfa339f3627ba https://conda.anaconda.org/conda-forge/linux-64/coverage-7.3.1-py311h459d7ec_0.conda#d23df37f3a595e8ffca99642ab6df3eb https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.6.0-h00ab1b0_0.conda#364c6ae36c4e36fcbd4d273cf4db78af https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_0.conda#5c416db47b7816e437eaf0d46e5c3a3d @@ -298,14 +293,16 @@ https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.0-pyhd8ed1ab_0.conda#1cd https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2#c8490ed5c70966d232fdd389d0dbed37 https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.2.2-pyhd8ed1ab_0.tar.bz2#243f63592c8e449f40cd42eb5cf32f40 https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 +https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-18_linux64_openblas.conda#93dd9ab275ad888ed8113953769af78c https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h74d50f4_7.conda#3453ac94a99ad9daf17e8a313d274567 -https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h840a212_1.conda#03c225a73835f5aa68c13e62eb360406 +https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h8d7e28b_2.conda#ed3cd026aa12259ce96c0552873705c9 +https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-18_linux64_openblas.conda#a1244707531e5b143c420c70573c8ec5 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.6-pyhd8ed1ab_0.tar.bz2#b21613793fcc81d944c76c9f2864a7de https://conda.anaconda.org/conda-forge/linux-64/mypy-1.5.1-py311h459d7ec_0.conda#c99b944502de2ce602b7d666df977a0c https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.8.0-pyhd8ed1ab_0.conda#2a75b296096adabbabadd5e9782e5fcc -https://conda.anaconda.org/conda-forge/noarch/partd-1.4.0-pyhd8ed1ab_0.conda#721dab5803ea92ce02ddc4ee50aa0c48 +https://conda.anaconda.org/conda-forge/noarch/partd-1.4.0-pyhd8ed1ab_1.conda#6ceb4e000cbe0b56b290180aea8520e8 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.8.0-pyh1a96a4e_2.tar.bz2#330448ce4403cc74990ac07c555942a1 https://conda.anaconda.org/conda-forge/linux-64/pillow-10.0.0-py311h0b84326_0.conda#4b24acdc1fbbae9da03147e7d2cf8c8a https://conda.anaconda.org/conda-forge/noarch/pip-23.2.1-pyhd8ed1ab_0.conda#e2783aa3f9235225eec92f9081c5b801 @@ -315,35 +312,32 @@ https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.3.0-pyhd8ed1ab_0.cond https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.2-pyhd8ed1ab_0.conda#6dd662ff5ac9a783e5c940ce9f3fe649 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 https://conda.anaconda.org/conda-forge/noarch/referencing-0.30.2-pyhd8ed1ab_0.conda#a33161b983172ba6ef69d5fc850650cd -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311he06c224_2.conda#10a1953d2f74d292b5de093ceea104b2 https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.2.1-pyhd8ed1ab_0.tar.bz2#7234c9eefff659501cd2fe0d2ede4d48 -https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.2-pyhd8ed1ab_0.conda#700fb06cd011d594305e3b487d5a96a2 -https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.7.1-hd8ed1ab_0.conda#f96688577f1faa58096d06a45136afa2 +https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.3-pyhd8ed1ab_0.conda#d2724da40562babd0322ab353b110225 +https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.8.0-hd8ed1ab_0.conda#384462e63262a527bda564fa2d9126c0 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 -https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.4-pyhd8ed1ab_0.conda#18badd8fa3648d1beb1fcc7f2e0f756e +https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.5-pyhd8ed1ab_0.conda#3bda70bbeb2920f44db5375af2e5fe38 https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-hac6953d_3.conda#297e6a75dc1b6a440cd341a85eab8a00 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.32.0-pyhd8ed1ab_0.conda#6d2425548b0293a225ca4febd80feaa3 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.14-h1678ad6_3.conda#5418f1a44b5bc75892d898e8fb5d180b -https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_0.conda#43a71a823583d75308eaf3a06c8f150b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.17-h1678ad6_0.conda#e99777ef77b880a59b6caf4405360694 https://conda.anaconda.org/conda-forge/linux-64/compilers-1.6.0-ha770c72_0.conda#e2259de4640a51a28c21931ae98e4975 -https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.3-py311h63ff55d_0.conda#cc8ad641cab65dfe59caddbc23a1aeca +https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.4-py311h63ff55d_0.conda#2b14cd05541532521196b0d2e0291ecf https://conda.anaconda.org/conda-forge/noarch/django-4.2.5-pyhd8ed1ab_0.conda#5af47510c844ef54896d6b3ea424b82b https://conda.anaconda.org/conda-forge/noarch/flake8-5.0.4-pyhd8ed1ab_0.tar.bz2#8079ea7dec0a917dd0cb6c257f7ea9ea https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h22adcc9_11.conda#514167b60f598eaed3f7a60e1dceb9ee -https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.35-pyhd8ed1ab_0.conda#e434463858a6060d5424d95fbd5fa2b8 -https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.2.0-h3d44ed6_0.conda#3c9bf4083e1a1be134b9a0c75cf7e635 +https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.36-pyhd8ed1ab_0.conda#a7c99acc18eea3d70224dd95dadcfd31 +https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.2.1-h3d44ed6_0.conda#98db5f8813f45e2b29766aff0e4a499c https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda#b279b07ce18058034e5b3606ba103a8b https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.7.1-pyhd8ed1ab_0.conda#7c27ea1bdbe520bb830dcadd59f55cbf https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-hcd42e92_5.conda#d871720bf750347506062ba23a91662d https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h80fb2b6_112.conda#a19fa6cacf80c8a366572853d5890eb4 https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h15f6e67_28.conda#bc9758e23157cb8362e60d3de06aa6fb -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.7.2-py311h54ef318_0.conda#2631a9e423855fb586c05f8a5ee8b177 -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.0-py311h320fe9a_0.conda#7f35501e126df510b250ad893482ef45 +https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.0-py311h64a7726_0.conda#bf16a9f625126e378302f08e7ed67517 https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda#0809187ef9b89a3d94a5c24d13936236 https://conda.anaconda.org/conda-forge/linux-64/poppler-23.08.0-hd18248d_0.conda#59a093146aa911da2ca056c1197e3e41 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.0-py311ha169711_1.conda#92633556d37e88ce45193374d408072c +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311ha169711_0.conda#ad4b6e9be79a89959bb6d7d308027ff2 https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.1.0-pyhd8ed1ab_0.conda#06eb685a3a0b146347a58dda979485da https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.0.1-pyhd8ed1ab_0.conda#9da651d84c73bac482cae51613a4d4d6 https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.0.0-pyhd8ed1ab_1.conda#8bdcc0f401561213821bf67513abeeff @@ -354,21 +348,20 @@ https://conda.anaconda.org/conda-forge/noarch/rdflib-7.0.0-pyhd8ed1ab_0.conda#44 https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda#a30144e4156cdbb236f99ebb49828f8b https://conda.anaconda.org/conda-forge/noarch/requirements-detector-1.2.2-pyhd8ed1ab_0.conda#6626918380d99292df110f3c91b6e5ec https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.2-pyhd8ed1ab_0.conda#e7df0fdd404616638df5ece6e69ba7af -https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h84d19f0_1.conda#1cc4e61dc7ca15a570e733a6e20a7b33 +https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h8c794c1_3.conda#7de728789b0aba16018f726dc5ddbec2 https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h4dd048b_3.tar.bz2#dbfea4376856bf7bd2121e719cf816e5 https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.6-pyhd8ed1ab_0.conda#078979d33523cb477bd1916ce41aacc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.23.1-h40cdbb9_0.conda#db54cd72dfe4764a6d11cc914099ec8e -https://conda.anaconda.org/conda-forge/noarch/bokeh-3.2.2-pyhd8ed1ab_0.conda#30488151f591379db656250b3f5fc0c6 -https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_0.conda#1271b2375735e2aaa6d6770dbe2ad087 -https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.9.1-pyhd8ed1ab_0.conda#e420da4262d61ab10156219edbabb54b +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.23.1-hffbee3f_1.conda#e56fcd606b5311dc13c70279b359f13b +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h4c7f6c3_1.tar.bz2#c7e54004ffd03f8db0a58ab949f2a00b +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.1-py311h9547e67_0.conda#db5b3b0093d0d4565e5c89578108402e +https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.9.2-pyhd8ed1ab_0.conda#cce7eeb7eda0124af186a5e9ce9b0fca https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe -https://conda.anaconda.org/conda-forge/noarch/identify-2.5.27-pyhd8ed1ab_0.conda#6fbde8d3bdd1874132a1b26a3554b22c -https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.0-pyhd8ed1ab_1.conda#d442886dffcee45604595fea2ad3a181 +https://conda.anaconda.org/conda-forge/noarch/identify-2.5.29-pyhd8ed1ab_0.conda#5bdbb1cb692649720b60f261b41760cd +https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.1-pyhd8ed1ab_0.conda#78aff5d2af74e6537c1ca73017f01f4f https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.3.1-py311h38be061_0.conda#0cf8259b01ede82c76007996f73f89ed -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.1-h880a63b_9.conda#6e41df426ad7c3153554297f57b9017d -https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.2-h323ed7e_1.conda#b85c17750b1dce1e97d976edf079dd8b https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hacb5139_102.conda#487a1c19dd3eacfd055ad614e9acde87 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311he8ad708_102.conda#b48083ba918347f30efa94f7dc694919 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.1-py311h320fe9a_0.conda#1692362ba82f0556099f0143f7842de3 https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-ha41ecd1_2.conda#1a66c10f6a0da3dbd2f3a68127e7f6a0 https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda#5936894aade8240c867d292aa0d980c6 https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.39-pyha770c72_0.conda#a4986c6bb5b0d05a38855b0880a5f425 @@ -376,53 +369,60 @@ https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.5-pyhd8ed1ab_0.conda#3 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.2.0-pyhd8ed1ab_1.conda#34f7d568bf59d18e3fef8c405cbece21 https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.1.0-pyhd8ed1ab_0.conda#57b89064c125bb9d0e533e018c3eb17a +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py311h64a7726_1.conda#58af16843fc4469770bdbaf45d3a19de +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311he06c224_2.conda#10a1953d2f74d292b5de093ceea104b2 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.4-pyhd8ed1ab_0.conda#c3feaf947264a59a125e8c26e98c3c5a -https://conda.anaconda.org/conda-forge/noarch/xarray-2023.8.0-pyhd8ed1ab_0.conda#a8104cede521616573e228c27f9edc97 https://conda.anaconda.org/conda-forge/noarch/yapf-0.40.1-pyhd8ed1ab_0.conda#f269942e802d5e148632143d4c37acc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.156-h8bde0db_1.conda#08924ffd0c6ef3ff540238960e31673b -https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 -https://conda.anaconda.org/conda-forge/noarch/distributed-2023.9.1-pyhd8ed1ab_0.conda#e3841f3799c666f8705a2aa6604af294 +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.156-he6c2984_2.conda#897c30dedccac6d1f73dd52b14e8e70f +https://conda.anaconda.org/conda-forge/noarch/bokeh-3.2.2-pyhd8ed1ab_0.conda#30488151f591379db656250b3f5fc0c6 +https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_0.conda#43a71a823583d75308eaf3a06c8f150b +https://conda.anaconda.org/conda-forge/noarch/distributed-2023.9.2-pyhd8ed1ab_0.conda#ddb4fd6105b4005b312625cef210ba67 https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_h9e768e6_3.conda#c330e87e698bae8e7381c0315cf25dd0 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.1-py311h815a124_9.conda#e026f17deff5512eeb5119b0e6ba9103 +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.2-py311h815a124_1.conda#19c9d9ac47f4ac001967ce16d5831886 https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.3.1-pyhd8ed1ab_0.conda#b7cc0981484fcb6390e6d341e55618b3 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda#620e754f4344f4c27259ff460a2b9c50 +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.0-py311h54ef318_0.conda#b67672c2f39ef2912a1814e29e42c7ca https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.2-pyhd8ed1ab_0.conda#61ba076de6530d9301a0053b02f093d2 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311he8ad708_102.conda#b48083ba918347f30efa94f7dc694919 https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d -https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.4.0-pyha770c72_0.conda#f0fe759dc1dc02722c15cfb5faa1172b +https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.4.0-pyha770c72_1.conda#3fb5ba328a77c9fd71197a46e7f2469a https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.39-hd8ed1ab_0.conda#4bbbe67d5df19db30f04b8e344dc9976 https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_0.conda#3a00b1b08d8c01b1a3bfa686b9152df2 -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py311h64a7726_0.conda#18d094fb8e4ac52f93a4f4857a8f1e8f +https://conda.anaconda.org/conda-forge/noarch/xarray-2023.8.0-pyhd8ed1ab_0.conda#a8104cede521616573e228c27f9edc97 +https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_0.conda#1271b2375735e2aaa6d6770dbe2ad087 +https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.8.2-pyhd8ed1ab_0.conda#cc344a296a41369bcb05f7216661cec8 https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyh1a96a4e_2.tar.bz2#64068564a9c2932bf30e9b4ec567927d https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda#ddcf387719b2e44df0cc4dd467643951 https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.4-py311hbac4ec9_0.conda#1d3445f5f7fa002a1c149c405376f012 https://conda.anaconda.org/conda-forge/linux-64/graphviz-8.1.0-h28d9a01_0.conda#33628e0e3de7afd2c8172f76439894cb https://conda.anaconda.org/conda-forge/noarch/ipython-8.15.0-pyh0d859eb_0.conda#6392e665cbdaa780ca2b7a01ac34bb4b -https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc1f660bf455c239adaabf56b91dc9 -https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-h1ed0495_3_cpu.conda#4abe40cdbe46985f6d90840e58338883 +https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-h1935d02_4_cpu.conda#f5efd1ff369209712c6277fd2f3b6c03 https://conda.anaconda.org/conda-forge/noarch/nbclient-0.8.0-pyhd8ed1ab_0.conda#e78da91cf428faaf05701ce8cc8f2f9b -https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.3-pyhd8ed1ab_0.conda#de9f9392273a1e6095930a21561a37e9 +https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 https://conda.anaconda.org/conda-forge/noarch/pylint-django-2.5.3-pyhd8ed1ab_0.tar.bz2#00d8853fb1f87195722ea6a582cc9b56 https://conda.anaconda.org/conda-forge/noarch/pylint-flask-0.6-py_0.tar.bz2#5a9afd3d0a61b08d59eed70fab859c1b +https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc1f660bf455c239adaabf56b91dc9 https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.8.0-pyhd8ed1ab_0.conda#62345c9e24f898bf492979be84a6eb0a https://conda.anaconda.org/conda-forge/noarch/prospector-1.10.2-pyhd8ed1ab_0.conda#2c536985982f7e531df8d640f554008a -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_3_cpu.conda#5791daab9a8c6c383f91a3f11460d384 +https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.3-pyhd8ed1ab_0.conda#de9f9392273a1e6095930a21561a37e9 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_4_cpu.conda#cf1ad9f013404a12bd0c823ca9b99742 https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_3.tar.bz2#64a77de29fde80aef5013ddf5e62a564 -https://conda.anaconda.org/conda-forge/noarch/dask-2023.9.1-pyhd8ed1ab_0.conda#7192f8f126e50d6d0274fbbf4e73e9cf +https://conda.anaconda.org/conda-forge/noarch/dask-2023.9.2-pyhd8ed1ab_0.conda#29e33df59c9eac1a599b9cd18d54b4d3 https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.8.0-pyhd8ed1ab_0.conda#1dba1a577df2625a24667612a069e91c https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.8.0-pyhd8ed1ab_0.conda#56e85460d22fa7d4fb06300f785dd1e1 https://conda.anaconda.org/conda-forge/noarch/nbconvert-7.8.0-pyhd8ed1ab_0.conda#43bce95e8c474dd21d7ed5de8b4806f7 https://conda.anaconda.org/conda-forge/noarch/autodocsumm-0.2.6-pyhd8ed1ab_0.tar.bz2#4409dd7e06a62c3b2aa9e96782c49c6d https://conda.anaconda.org/conda-forge/noarch/nbsphinx-0.9.3-pyhd8ed1ab_0.conda#0dbaa7d08d3d79b2a1a4dd6a02cc4581 -https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.13.3-pyhd8ed1ab_0.conda#07aca5f2dea315dcc16680d6891e9056 +https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.14.1-pyhd8ed1ab_0.conda#78153addf629c51fab775ef360012ca3 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-1.0.7-pyhd8ed1ab_0.conda#aebfabcb60c33a89c1f9290cab49bc93 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-1.0.5-pyhd8ed1ab_0.conda#ebf08f5184d8eaa486697bc060031953 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.0.4-pyhd8ed1ab_0.conda#a9a89000dfd19656ad004b937eeb6828 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-1.0.6-pyhd8ed1ab_0.conda#cf5c9649272c677a964a7313279e3a9b -https://conda.anaconda.org/conda-forge/noarch/sphinx-7.2.5-pyhd8ed1ab_0.conda#bc7881d37168b6731affdd89a2433c64 +https://conda.anaconda.org/conda-forge/noarch/sphinx-7.2.6-pyhd8ed1ab_0.conda#bbfd1120d1824d2d073bc65935f0e4c0 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-1.1.9-pyhd8ed1ab_0.conda#0612e497d7860728f2cda421ea2aec09 From a6f257a3f29b77e9449901ecbdb7b89c0177d083 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 26 Sep 2023 17:45:09 +0200 Subject: [PATCH 05/72] Avoid a crash if dataset has supplementary variables (#2198) Co-authored-by: Valeriu Predoi --- esmvalcore/_recipe/from_datasets.py | 19 +++++++++++++++++++ tests/unit/recipe/test_from_datasets.py | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index c54931806f..0e454c7350 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -160,6 +160,23 @@ def _group_identical_facets(variable: Mapping[str, Any]) -> Recipe: return result +class _SortableDict(dict): + """A `dict` class that can be sorted.""" + + def __lt__(self, other): + return tuple(self.items()) < tuple(other.items()) + + +def _change_dict_type(item, dict_type): + """Change the dict type in a nested structure.""" + change_dict_type = partial(_change_dict_type, dict_type=dict_type) + if isinstance(item, dict): + return dict_type((k, change_dict_type(v)) for k, v in item.items()) + if isinstance(item, (list, tuple, set)): + return type(item)(change_dict_type(elem) for elem in item) + return item + + def _group_ensemble_members(dataset_facets: Iterable[Facets]) -> list[Facets]: """Group ensemble members. @@ -171,6 +188,7 @@ def grouper(facets): return tuple((k, facets[k]) for k in sorted(facets) if k != 'ensemble') result = [] + dataset_facets = _change_dict_type(dataset_facets, _SortableDict) dataset_facets = sorted(dataset_facets, key=grouper) for group_facets, group in itertools.groupby(dataset_facets, key=grouper): ensembles = [f['ensemble'] for f in group if 'ensemble' in f] @@ -181,6 +199,7 @@ def grouper(facets): facets = dict(group_facets) facets['ensemble'] = ensemble result.append(facets) + result = _change_dict_type(result, dict) return result diff --git a/tests/unit/recipe/test_from_datasets.py b/tests/unit/recipe/test_from_datasets.py index 137f55f0c1..c2032724a4 100644 --- a/tests/unit/recipe/test_from_datasets.py +++ b/tests/unit/recipe/test_from_datasets.py @@ -8,6 +8,7 @@ _group_ensemble_names, _group_identical_facets, _move_one_level_up, + _SortableDict, _to_frozen, datasets_to_recipe, ) @@ -136,6 +137,12 @@ def test_supplementary_datasets_to_recipe(): assert datasets_to_recipe([dataset]) == recipe +def test_sortable_dict(): + assert _SortableDict({'a': 1}) < _SortableDict({'a': 2}) + assert _SortableDict({'a': 1}) < _SortableDict({'a': 1, 'b': 1}) + assert _SortableDict({'a': 1}) < _SortableDict({'b': 1}) + + def test_datasets_to_recipe_group_ensembles(): datasets = [ Dataset( From 10bd77633672f9120730362b1d77c69d62ce828a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 27 Sep 2023 10:37:34 +0200 Subject: [PATCH 06/72] Remove deprecated way of calling ``esmvalcore.cmor.table.read_cmor_tables`` (#2201) Co-authored-by: Valeriu Predoi --- esmvalcore/cmor/table.py | 36 +++---------------- .../integration/cmor/test_read_cmor_tables.py | 11 ++---- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 680e7289a1..504a31c137 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -11,8 +11,6 @@ import json import logging import os -import tempfile -import warnings from collections import Counter from functools import lru_cache, total_ordering from pathlib import Path @@ -20,7 +18,7 @@ import yaml -from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError +from esmvalcore.exceptions import RecipeError logger = logging.getLogger(__name__) @@ -94,35 +92,11 @@ def read_cmor_tables(cfg_developer: Optional[Path] = None) -> None: ---------- cfg_developer: Path to config-developer.yml file. - - Prior to v2.8.0 `cfg_developer` was an :obj:`dict` with the contents - of config-developer.yml. This is deprecated and support will be - removed in v2.10.0. """ - if isinstance(cfg_developer, dict): - warnings.warn( - "Using the `read_cmor_tables` file with a dictionary as argument " - "has been deprecated in ESMValCore version 2.8.0 and is " - "scheduled for removal in version 2.10.0. " - "Please use the path to the config-developer.yml file instead.", - ESMValCoreDeprecationWarning, - ) - with tempfile.NamedTemporaryFile( - mode='w', - encoding='utf-8', - delete=False, - ) as file: - yaml.safe_dump(cfg_developer, file) - cfg_file = Path(file.name) - else: - cfg_file = cfg_developer - if cfg_file is None: - cfg_file = Path(__file__).parents[1] / 'config-developer.yml' - mtime = cfg_file.stat().st_mtime - cmor_tables = _read_cmor_tables(cfg_file, mtime) - if isinstance(cfg_developer, dict): - # clean up the temporary file - cfg_file.unlink() + if cfg_developer is None: + cfg_developer = Path(__file__).parents[1] / 'config-developer.yml' + mtime = cfg_developer.stat().st_mtime + cmor_tables = _read_cmor_tables(cfg_developer, mtime) CMOR_TABLES.clear() CMOR_TABLES.update(cmor_tables) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index facb94f96b..4a50e2c76f 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,6 +1,5 @@ from pathlib import Path -import pytest import yaml from esmvalcore.cmor.table import CMOR_TABLES @@ -45,15 +44,11 @@ def test_read_cmor_tables(): assert table.strict is False -@pytest.mark.parametrize('behaviour', ['current', 'deprecated']) -def test_read_custom_cmor_tables(tmp_path, behaviour): +def test_read_custom_cmor_tables(tmp_path): """Test reading of custom CMOR tables.""" cfg_file = tmp_path / 'config-developer.yml' - if behaviour == 'deprecated': - cfg_file = CUSTOM_CFG_DEVELOPER - else: - with cfg_file.open('w', encoding='utf-8') as file: - yaml.safe_dump(CUSTOM_CFG_DEVELOPER, file) + with cfg_file.open('w', encoding='utf-8') as file: + yaml.safe_dump(CUSTOM_CFG_DEVELOPER, file) read_cmor_tables(cfg_file) From ac31d5352e5dc79a3f75eba809069a8731c432ea Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:57:19 +0200 Subject: [PATCH 07/72] Clearly separate fixes and CMOR checks (#2157) --- esmvalcore/cmor/_fixes/cmip6/iitm_esm.py | 4 +- esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py | 4 +- esmvalcore/cmor/_fixes/fix.py | 728 +++++++++++++- esmvalcore/cmor/_utils.py | 219 +++++ esmvalcore/cmor/check.py | 728 +++++--------- esmvalcore/cmor/fix.py | 254 ++--- esmvalcore/cmor/table.py | 97 +- esmvalcore/preprocessor/_time.py | 6 +- .../cmor/_fixes/cesm/test_cesm2.py | 2 + .../cmor/_fixes/cmip5/test_access1_0.py | 9 +- .../cmor/_fixes/cmip5/test_access1_3.py | 9 +- .../cmor/_fixes/cmip5/test_bcc_csm1_1.py | 15 +- .../cmor/_fixes/cmip5/test_bcc_csm1_1_m.py | 16 +- .../cmor/_fixes/cmip5/test_bnu_esm.py | 24 +- .../cmor/_fixes/cmip5/test_canesm2.py | 6 +- .../cmor/_fixes/cmip5/test_ccsm4.py | 3 +- .../cmor/_fixes/cmip5/test_cesm1_bgc.py | 9 +- .../cmor/_fixes/cmip5/test_cesm1_cam5.py | 3 +- .../cmor/_fixes/cmip5/test_cesm1_fastchem.py | 3 +- .../cmor/_fixes/cmip5/test_cesm1_waccm.py | 3 +- .../cmor/_fixes/cmip5/test_cnrm_cm5.py | 7 +- .../cmor/_fixes/cmip5/test_csiro_mk3_6_0.py | 4 +- .../cmor/_fixes/cmip5/test_ec_earth.py | 16 +- .../cmor/_fixes/cmip5/test_fgoals_g2.py | 6 +- .../cmor/_fixes/cmip5/test_fgoals_s2.py | 3 +- .../cmor/_fixes/cmip5/test_fio_esm.py | 11 +- .../cmor/_fixes/cmip5/test_gfdl_cm2p1.py | 20 +- .../cmor/_fixes/cmip5/test_gfdl_cm3.py | 7 +- .../cmor/_fixes/cmip5/test_gfdl_esm2g.py | 27 +- .../cmor/_fixes/cmip5/test_gfdl_esm2m.py | 17 +- .../cmor/_fixes/cmip5/test_giss_e2_h.py | 4 +- .../cmor/_fixes/cmip5/test_giss_e2_r.py | 4 +- .../cmor/_fixes/cmip5/test_hadgem2_cc.py | 7 +- .../cmor/_fixes/cmip5/test_hadgem2_es.py | 7 +- .../cmor/_fixes/cmip5/test_inmcm4.py | 9 +- .../cmor/_fixes/cmip5/test_ipsl_cm5a_lr.py | 4 +- .../cmor/_fixes/cmip5/test_ipsl_cm5a_mr.py | 4 +- .../cmor/_fixes/cmip5/test_ipsl_cm5b_lr.py | 4 +- .../cmor/_fixes/cmip5/test_miroc5.py | 11 +- .../cmor/_fixes/cmip5/test_miroc_esm.py | 13 +- .../cmor/_fixes/cmip5/test_miroc_esm_chem.py | 5 +- .../cmor/_fixes/cmip5/test_mpi_esm_lr.py | 5 +- .../cmor/_fixes/cmip5/test_mpi_esm_mr.py | 4 +- .../cmor/_fixes/cmip5/test_mpi_esm_p.py | 4 +- .../cmor/_fixes/cmip5/test_mri_cgcm3.py | 7 +- .../cmor/_fixes/cmip5/test_mri_esm1.py | 5 +- .../cmor/_fixes/cmip5/test_noresm1_m.py | 4 +- .../cmor/_fixes/cmip5/test_noresm1_me.py | 9 +- .../cmor/_fixes/cmip6/test_access_cm2.py | 7 +- .../cmor/_fixes/cmip6/test_access_esm1_5.py | 11 +- .../cmor/_fixes/cmip6/test_awi_cm_1_1_mr.py | 3 +- .../cmor/_fixes/cmip6/test_awi_esm_1_1_lr.py | 3 +- .../cmor/_fixes/cmip6/test_bcc_csm2_mr.py | 16 +- .../cmor/_fixes/cmip6/test_bcc_esm1.py | 14 +- .../cmor/_fixes/cmip6/test_cams_csm1_0.py | 8 +- .../cmor/_fixes/cmip6/test_canesm5.py | 5 +- .../cmor/_fixes/cmip6/test_canesm5_canoe.py | 6 +- .../cmor/_fixes/cmip6/test_cas_esm2_0.py | 3 +- .../cmor/_fixes/cmip6/test_cesm2.py | 17 +- .../cmor/_fixes/cmip6/test_cesm2_fv2.py | 13 +- .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 13 +- .../cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py | 15 +- .../cmor/_fixes/cmip6/test_ciesm.py | 3 +- .../cmor/_fixes/cmip6/test_cmcc_cm2_sr5.py | 3 +- .../cmor/_fixes/cmip6/test_cnrm_cm6_1.py | 20 +- .../cmor/_fixes/cmip6/test_cnrm_cm6_1_hr.py | 8 +- .../cmor/_fixes/cmip6/test_cnrm_esm2_1.py | 21 +- .../cmor/_fixes/cmip6/test_e3sm_1_0.py | 3 +- .../cmor/_fixes/cmip6/test_ec_earth3_veg.py | 9 +- .../_fixes/cmip6/test_ec_earth3_veg_lr.py | 4 +- .../cmor/_fixes/cmip6/test_fgoals_f3_l.py | 16 +- .../cmor/_fixes/cmip6/test_fgoals_g3.py | 7 +- .../cmor/_fixes/cmip6/test_fio_esm_2_0.py | 5 +- .../cmor/_fixes/cmip6/test_gfdl_cm4.py | 21 +- .../cmor/_fixes/cmip6/test_gfdl_esm4.py | 7 +- .../cmor/_fixes/cmip6/test_giss_e2_1_g.py | 10 +- .../cmor/_fixes/cmip6/test_giss_e2_1_h.py | 8 +- .../cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py | 9 +- .../cmor/_fixes/cmip6/test_icon_esm_lr.py | 3 +- .../cmor/_fixes/cmip6/test_iitm_esm.py | 3 +- .../cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py | 6 +- .../cmor/_fixes/cmip6/test_kace_1_0_g.py | 11 +- .../cmor/_fixes/cmip6/test_kiost_esm.py | 8 +- .../cmor/_fixes/cmip6/test_mcm_ua_1_0.py | 9 +- .../cmor/_fixes/cmip6/test_miroc6.py | 10 +- .../cmor/_fixes/cmip6/test_miroc_es2l.py | 8 +- .../cmor/_fixes/cmip6/test_mpi_esm1_2_lr.py | 7 +- .../cmor/_fixes/cmip6/test_mpi_esm_1_2_ham.py | 7 +- .../cmor/_fixes/cmip6/test_mri_esm2_0.py | 8 +- .../cmor/_fixes/cmip6/test_nesm3.py | 8 +- .../cmor/_fixes/cmip6/test_noresm2_lm.py | 9 +- .../cmor/_fixes/cmip6/test_noresm2_mm.py | 7 +- .../cmor/_fixes/cmip6/test_sam0_unicon.py | 10 +- .../cmor/_fixes/cmip6/test_taiesm1.py | 7 +- .../cmor/_fixes/cmip6/test_ukesm1_0_ll.py | 9 +- .../integration/cmor/_fixes/emac/test_emac.py | 137 +-- .../integration/cmor/_fixes/icon/test_icon.py | 25 +- .../cmor/_fixes/ipslcm/test_ipsl_cm6.py | 3 +- .../cmor/_fixes/native6/test_era5.py | 17 +- .../cmor/_fixes/obs4mips/test_airs_2_1.py | 3 +- .../cmor/_fixes/obs4mips/test_ssmi.py | 3 +- .../cmor/_fixes/obs4mips/test_ssmi_meris.py | 3 +- tests/integration/cmor/_fixes/test_fix.py | 65 +- tests/integration/cmor/test_fix.py | 909 ++++++++++++++++++ tests/integration/cmor/test_table.py | 41 + tests/unit/cmor/test_cmor_check.py | 476 +++------ tests/unit/cmor/test_fix.py | 133 +-- tests/unit/cmor/test_generic_fix.py | 237 +++++ tests/unit/cmor/test_utils.py | 60 ++ tests/unit/test_cmor_api.py | 113 +++ 110 files changed, 3551 insertions(+), 1462 deletions(-) create mode 100644 esmvalcore/cmor/_utils.py create mode 100644 tests/integration/cmor/test_fix.py create mode 100644 tests/unit/cmor/test_generic_fix.py create mode 100644 tests/unit/cmor/test_utils.py diff --git a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py index 6ee5833c26..a6dee710c9 100644 --- a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py +++ b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor.check import _get_time_bounds +from esmvalcore.cmor._fixes.fix import get_time_bounds from ..common import OceanFixGrid from ..fix import Fix @@ -33,7 +33,7 @@ def fix_metadata(self, cubes): for cube in cubes: freq = self.extra_facets["frequency"] time = cube.coord("time", dim_coords=True) - bounds = _get_time_bounds(time, freq) + bounds = get_time_bounds(time, freq) if np.any(bounds != time.bounds): time.bounds = bounds logger.warning( diff --git a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py index f2a0a1b7ac..5e27377b0d 100644 --- a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py +++ b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor.check import _get_time_bounds +from esmvalcore.cmor._fixes.fix import get_time_bounds from ..common import ClFixHybridHeightCoord, OceanFixGrid from ..fix import Fix @@ -39,7 +39,7 @@ def fix_metadata(self, cubes): for cube in cubes: freq = self.extra_facets["frequency"] time = cube.coord("time", dim_coords=True) - bounds = _get_time_bounds(time, freq) + bounds = get_time_bounds(time, freq) if np.any(bounds != time.bounds): time.bounds = bounds logger.warning( diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 2deeb0d6d3..4b5163d4a8 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -3,17 +3,36 @@ import importlib import inspect +import logging import tempfile +from collections.abc import Sequence +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Optional +import numpy as np +from cf_units import Unit +from iris.coords import Coord, CoordExtent from iris.cube import Cube, CubeList - -from ..table import CMOR_TABLES +from iris.exceptions import UnitConversionError +from iris.util import reverse + +from esmvalcore.cmor._utils import ( + _get_alternative_generic_lev_coord, + _get_generic_lev_coord_names, + _get_new_generic_level_coord, + _get_simplified_calendar, + _get_single_cube, + _is_unstructured_grid, +) +from esmvalcore.cmor.table import get_var_info +from esmvalcore.iris_helpers import date2num if TYPE_CHECKING: - from ...config import Session - from ..table import VariableInfo + from esmvalcore.cmor.table import CoordinateInfo, VariableInfo + from esmvalcore.config import Session + +logger = logging.getLogger(__name__) class Fix: @@ -24,19 +43,23 @@ def __init__( vardef: VariableInfo, extra_facets: Optional[dict] = None, session: Optional[Session] = None, + frequency: Optional[str] = None, ) -> None: """Initialize fix object. Parameters ---------- vardef: - CMOR table entry. + CMOR table entry of the variable. extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. session: Current session which includes configuration and directory information. + frequency: + Expected frequency of the variable. If not given, use the one from + the CMOR table entry of the variable. """ self.vardef = vardef @@ -44,6 +67,9 @@ def __init__( extra_facets = {} self.extra_facets = extra_facets self.session = session + if frequency is None and self.vardef is not None: + frequency = self.vardef.frequency + self.frequency = frequency def fix_file( self, @@ -76,7 +102,7 @@ def fix_file( """ return filepath - def fix_metadata(self, cubes: CubeList) -> CubeList: + def fix_metadata(self, cubes: Sequence[Cube]) -> Sequence[Cube]: """Apply fixes to the metadata of the cube. Changes applied here must not require data loading. @@ -90,7 +116,7 @@ def fix_metadata(self, cubes: CubeList) -> CubeList: Returns ------- - iris.cube.CubeList + Iterable[iris.cube.Cube] Fixed cubes. They can be different instances. """ @@ -163,6 +189,7 @@ def get_fixes( short_name: str, extra_facets: Optional[dict] = None, session: Optional[Session] = None, + frequency: Optional[str] = None, ) -> list: """Get the fixes that must be applied for a given dataset. @@ -177,6 +204,8 @@ def get_fixes( before checking because it is not possible to use the character '-' in python names. + In addition, generic fixes for all datasets are added. + Parameters ---------- project: @@ -193,6 +222,9 @@ def get_fixes( session: Current session which includes configuration and directory information. + frequency: + Expected frequency of the variable. If not given, use the one from + the CMOR table entry of the variable. Returns ------- @@ -200,8 +232,7 @@ def get_fixes( Fixes to apply for the given data. """ - cmor_table = CMOR_TABLES[project] - vardef = cmor_table.get_variable(mip, short_name) + vardef = get_var_info(project, mip, short_name) project = project.replace('-', '_').lower() dataset = dataset.replace('-', '_').lower() @@ -238,9 +269,24 @@ def get_fixes( ) for fix_name in (short_name, mip.lower(), 'allvars'): if fix_name in classes: - fixes.append(classes[fix_name]( - vardef, extra_facets=extra_facets, session=session - )) + fixes.append( + classes[fix_name]( + vardef, + extra_facets=extra_facets, + session=session, + frequency=frequency, + ) + ) + + # Always perform generic fixes for all datasets + fixes.append( + GenericFix( + vardef, # type: ignore + extra_facets=extra_facets, + session=session, + frequency=frequency, + ) + ) return fixes @@ -277,3 +323,661 @@ def get_fixed_filepath( else: output_dir.mkdir(parents=True, exist_ok=True) return output_dir / Path(filepath).name + + +def get_next_month(month: int, year: int) -> tuple[int, int]: + """Get next month and year. + + Parameters + ---------- + month: + Current month. + year: + Current year. + + Returns + ------- + tuple[int, int] + Next month and next year. + + """ + if month != 12: + return month + 1, year + return 1, year + 1 + + +def get_time_bounds(time: Coord, freq: str): + """Get bounds for time coordinate. + + Parameters + ---------- + time: + Time coordinate. + freq: + Frequency. + + Returns + ------- + np.ndarray + Time bounds + + Raises + ------ + NotImplementedError + Non-supported frequency is given. + + """ + bounds = [] + dates = time.units.num2date(time.points) + for step, date in enumerate(dates): + month = date.month + year = date.year + if freq in ['mon', 'mo']: + next_month, next_year = get_next_month(month, year) + min_bound = date2num(datetime(year, month, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), + time.units, time.dtype) + elif freq == 'yr': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), + time.units, time.dtype) + elif freq == 'dec': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), + time.units, time.dtype) + else: + delta = { + 'day': 12.0 / 24, + '6hr': 3.0 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, + } + if freq not in delta: + raise NotImplementedError( + f"Cannot guess time bounds for frequency '{freq}'" + ) + point = time.points[step] + min_bound = point - delta[freq] + max_bound = point + delta[freq] + bounds.append([min_bound, max_bound]) + + return np.array(bounds) + + +class GenericFix(Fix): + """Class providing generic fixes for all datasets.""" + + def fix_metadata(self, cubes: Sequence[Cube]) -> CubeList: + """Fix cube metadata. + + Parameters + ---------- + cubes: + Cubes to be fixed. + + Returns + ------- + CubeList + Fixed cubes. + + """ + # Make sure the this fix also works when no extra_facets are given + if 'project' in self.extra_facets and 'dataset' in self.extra_facets: + dataset_str = ( + f"{self.extra_facets['project']}:" + f"{self.extra_facets['dataset']}" + ) + else: + dataset_str = None + + # The following fixes are designed to operate on the actual cube that + # corresponds to the variable. Thus, it needs to be assured (possibly + # by prior dataset-specific fixes) that the cubes here contain only one + # relevant cube. + cube = _get_single_cube( + cubes, self.vardef.short_name, dataset_str=dataset_str + ) + + cube = self._fix_standard_name(cube) + cube = self._fix_long_name(cube) + cube = self._fix_psu_units(cube) + cube = self._fix_units(cube) + + cube = self._fix_regular_coord_names(cube) + cube = self._fix_alternative_generic_level_coords(cube) + cube = self._fix_coords(cube) + cube = self._fix_time_coord(cube) + + return CubeList([cube]) + + def fix_data(self, cube: Cube) -> Cube: + """Fix cube data. + + Parameters + ---------- + cube: + Cube to be fixed. + + Returns + ------- + Cube + Fixed cube. + + """ + return cube + + @staticmethod + def _msg_suffix(cube: Cube) -> str: + """Get prefix for log messages.""" + if 'source_file' in cube.attributes: + return f"\n(for file {cube.attributes['source_file']})" + return f"\n(for variable {cube.var_name})" + + def _debug_msg(self, cube: Cube, msg: str, *args) -> None: + """Print debug message.""" + msg += self._msg_suffix(cube) + logger.debug(msg, *args) + + def _warning_msg(self, cube: Cube, msg: str, *args) -> None: + """Print debug message.""" + msg += self._msg_suffix(cube) + logger.warning(msg, *args) + + @staticmethod + def _set_range_in_0_360(array: np.ndarray) -> np.ndarray: + """Convert longitude coordinate to [0, 360].""" + return (array + 360.0) % 360.0 + + def _reverse_coord(self, cube: Cube, coord: Coord) -> tuple[Cube, Coord]: + """Reverse cube along a given coordinate.""" + if coord.ndim == 1: + cube = reverse(cube, cube.coord_dims(coord)) + coord = cube.coord(var_name=coord.var_name) + if coord.has_bounds(): + bounds = coord.core_bounds() + right_bounds = bounds[:-2, 1] + left_bounds = bounds[1:-1, 0] + if np.all(right_bounds != left_bounds): + coord.bounds = np.fliplr(bounds) + self._debug_msg( + cube, + "Coordinate %s values have been reversed", + coord.var_name, + ) + return (cube, coord) + + def _get_effective_units(self) -> str: + """Get effective units.""" + if self.vardef.units.lower() == 'psu': + return '1' + return self.vardef.units + + def _fix_units(self, cube: Cube) -> Cube: + """Fix cube units.""" + if self.vardef.units: + units = self._get_effective_units() + + # We use str(cube.units) in the following to catch `degrees` != + # `degrees_north` + if str(cube.units) != units: + old_units = cube.units + try: + cube.convert_units(units) + except (ValueError, UnitConversionError): + self._warning_msg( + cube, + "Failed to convert cube units from '%s' to '%s'", + old_units, + units, + ) + else: + self._warning_msg( + cube, + "Converted cube units from '%s' to '%s'", + old_units, + units, + ) + return cube + + def _fix_standard_name(self, cube: Cube) -> Cube: + """Fix standard_name.""" + # Do not change empty standard names + if not self.vardef.standard_name: + return cube + + if cube.standard_name != self.vardef.standard_name: + self._warning_msg( + cube, + "Standard name changed from '%s' to '%s'", + cube.standard_name, + self.vardef.standard_name, + ) + cube.standard_name = self.vardef.standard_name + + return cube + + def _fix_long_name(self, cube: Cube) -> Cube: + """Fix long_name.""" + # Do not change empty long names + if not self.vardef.long_name: + return cube + + if cube.long_name != self.vardef.long_name: + self._warning_msg( + cube, + "Long name changed from '%s' to '%s'", + cube.long_name, + self.vardef.long_name, + ) + cube.long_name = self.vardef.long_name + + return cube + + def _fix_psu_units(self, cube: Cube) -> Cube: + """Fix psu units.""" + if cube.attributes.get('invalid_units', '').lower() == 'psu': + cube.units = '1' + cube.attributes.pop('invalid_units') + self._debug_msg(cube, "Units converted from 'psu' to '1'") + return cube + + def _fix_regular_coord_names(self, cube: Cube) -> Cube: + """Fix regular (non-generic-level) coordinate names.""" + for cmor_coord in self.vardef.coordinates.values(): + if cmor_coord.generic_level: + continue # Ignore generic level coordinate in this function + if cube.coords(var_name=cmor_coord.out_name): + continue # Coordinate found -> fine here + if cube.coords(cmor_coord.standard_name): + cube_coord = cube.coord(cmor_coord.standard_name) + self._fix_cmip6_multidim_lat_lon_coord( + cube, cmor_coord, cube_coord + ) + return cube + + def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: + """Fix alternative generic level coordinates.""" + # Avoid overriding existing variable information + cmor_var_coordinates = self.vardef.coordinates.copy() + for (coord_name, cmor_coord) in cmor_var_coordinates.items(): + if not cmor_coord.generic_level: + continue # Ignore non-generic-level coordinates + if not cmor_coord.generic_lev_coords: + continue # Cannot fix anything without coordinate info + + # Extract names of the actual generic level coordinates present in + # the cube (e.g., `hybrid_height`, `standard_hybrid_sigma`) + (standard_name, out_name, name) = _get_generic_lev_coord_names( + cube, cmor_coord + ) + + # Make sure to update variable information with actual generic + # level coordinate if one has been found; this is necessary for + # subsequent fixes + if standard_name: + new_generic_level_coord = _get_new_generic_level_coord( + self.vardef, cmor_coord, coord_name, name + ) + self.vardef.coordinates[coord_name] = new_generic_level_coord + self._debug_msg( + cube, + "Generic level coordinate %s will be checked against %s " + "coordinate information", + coord_name, + name, + ) + + # If a generic level coordinate has been found, we don't need to + # look for alternatives + if standard_name or out_name: + continue + + # Search for alternative coordinates (i.e., regular level + # coordinates); if none found, do nothing + try: + (alternative_coord, + cube_coord) = _get_alternative_generic_lev_coord( + cube, coord_name, self.vardef.table_type + ) + except ValueError: # no alternatives found + continue + + # Fix alternative coord + (cube, cube_coord) = self._fix_coord( + cube, alternative_coord, cube_coord + ) + + return cube + + def _fix_cmip6_multidim_lat_lon_coord( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix CMIP6 multidimensional latitude and longitude coordinates.""" + is_cmip6_multidim_lat_lon = all([ + 'CMIP6' in self.vardef.table_type, + cube_coord.ndim > 1, + cube_coord.standard_name in ('latitude', 'longitude'), + ]) + if is_cmip6_multidim_lat_lon: + self._debug_msg( + cube, + "Multidimensional %s coordinate is not set in CMOR standard, " + "ESMValTool will change the original value of '%s' to '%s' to " + "match the one-dimensional case", + cube_coord.standard_name, + cube_coord.var_name, + cmor_coord.out_name, + ) + cube_coord.var_name = cmor_coord.out_name + + def _fix_coord_units( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix coordinate units.""" + if not cmor_coord.units: + return + + # We use str(cube_coord.units) in the following to catch `degrees` != + # `degrees_north` + if str(cube_coord.units) != cmor_coord.units: + old_units = cube_coord.units + try: + cube_coord.convert_units(cmor_coord.units) + except (ValueError, UnitConversionError): + self._warning_msg( + cube, + "Failed to convert units of coordinate %s from '%s' to " + "'%s'", + cmor_coord.out_name, + old_units, + cmor_coord.units, + ) + else: + self._warning_msg( + cube, + "Coordinate %s units '%s' converted to '%s'", + cmor_coord.out_name, + old_units, + cmor_coord.units, + ) + + def _fix_requested_coord_values( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix requested coordinate values.""" + if not cmor_coord.requested: + return + + # Cannot fix non-1D points + if cube_coord.core_points().ndim != 1: + return + + # Get requested CMOR values + try: + cmor_points = np.array(cmor_coord.requested, dtype=float) + except ValueError: + return + + # Align coordinate points with CMOR values if possible + if cube_coord.core_points().shape != cmor_points.shape: + return + atol = 1e-7 * np.mean(cmor_points) + align_coords = np.allclose( + cube_coord.core_points(), + cmor_points, + rtol=1e-7, + atol=atol, + ) + if align_coords: + cube_coord.points = cmor_points + self._debug_msg( + cube, + "Aligned %s points with CMOR points", + cmor_coord.out_name, + ) + + def _fix_longitude_0_360( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix longitude coordinate to be in [0, 360].""" + if not cube_coord.standard_name == 'longitude': + return (cube, cube_coord) + + # Only apply fixes when values are outside of valid range [0, 360] + inside_0_360 = all([ + cube_coord.core_points().min() >= 0.0, + cube_coord.core_points().max() <= 360.0, + ]) + if inside_0_360: + return (cube, cube_coord) + + # Cannot fix longitudes outside [-360, 720] + if np.any(cube_coord.core_points() < -360.0): + return (cube, cube_coord) + if np.any(cube_coord.core_points() > 720.0): + return (cube, cube_coord) + + # cube.intersection only works for cells with 0 or 2 bounds + # Note: nbounds==0 means there are no bounds given, nbounds==2 + # implies a regular grid with bounds in the grid direction, + # nbounds>2 implies an irregular grid with bounds given as vertices + # of the cell polygon. + if cube_coord.ndim == 1 and cube_coord.nbounds in (0, 2): + lon_extent = CoordExtent(cube_coord, 0.0, 360., True, False) + cube = cube.intersection(lon_extent) + else: + new_lons = cube_coord.core_points().copy() + new_lons = self._set_range_in_0_360(new_lons) + + if cube_coord.core_bounds() is None: + new_bounds = None + else: + new_bounds = cube_coord.core_bounds().copy() + new_bounds = self._set_range_in_0_360(new_bounds) + + new_coord = cube_coord.copy(new_lons, new_bounds) + dims = cube.coord_dims(cube_coord) + cube.remove_coord(cube_coord) + cube.add_aux_coord(new_coord, dims) + new_coord = cube.coord(var_name=cmor_coord.out_name) + self._debug_msg(cube, "Shifted longitude to [0, 360]") + + return (cube, new_coord) + + def _fix_coord_bounds( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> None: + """Fix coordinate bounds.""" + if cmor_coord.must_have_bounds != 'yes' or cube_coord.has_bounds(): + return + + # Skip guessing bounds for unstructured grids + if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + 'latitude', 'longitude'): + self._debug_msg( + cube, + "Will not guess bounds for coordinate %s of unstructured grid", + cube_coord.var_name, + ) + return + + try: + cube_coord.guess_bounds() + self._warning_msg( + cube, + "Added guessed bounds to coordinate %s", + cube_coord.var_name, + ) + except ValueError as exc: + self._warning_msg( + cube, + "Cannot guess bounds for coordinate %s: %s", + cube.var_name, + str(exc), + ) + + def _fix_coord_direction( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix coordinate direction (increasing vs. decreasing).""" + # Skip fix for a variety of reasons + if cube_coord.ndim > 1: + return (cube, cube_coord) + if cube_coord.dtype.kind == 'U': + return (cube, cube_coord) + if _is_unstructured_grid(cube) and cube_coord.standard_name in ( + 'latitude', 'longitude' + ): + return (cube, cube_coord) + if len(cube_coord.core_points()) == 1: + return (cube, cube_coord) + if not cmor_coord.stored_direction: + return (cube, cube_coord) + + # Fix coordinates with wrong direction + if cmor_coord.stored_direction == 'increasing': + if cube_coord.core_points()[0] > cube_coord.core_points()[1]: + (cube, cube_coord) = self._reverse_coord(cube, cube_coord) + elif cmor_coord.stored_direction == 'decreasing': + if cube_coord.core_points()[0] < cube_coord.core_points()[1]: + (cube, cube_coord) = self._reverse_coord(cube, cube_coord) + + return (cube, cube_coord) + + def _fix_time_units(self, cube: Cube, cube_coord: Coord) -> None: + """Fix time units in cube and attributes.""" + # Fix cube units + old_units = cube_coord.units + cube_coord.convert_units( + Unit( + 'days since 1850-1-1 00:00:00', + calendar=cube_coord.units.calendar, + ) + ) + simplified_cal = _get_simplified_calendar(cube_coord.units.calendar) + cube_coord.units = Unit( + cube_coord.units.origin, calendar=simplified_cal + ) + + # Fix units of time-related cube attributes + attrs = cube.attributes + parent_time = 'parent_time_units' + if parent_time in attrs: + if attrs[parent_time] in 'no parent': + pass + else: + try: + parent_units = Unit(attrs[parent_time], simplified_cal) + except ValueError: + pass + else: + attrs[parent_time] = 'days since 1850-1-1 00:00:00' + + branch_parent = 'branch_time_in_parent' + if branch_parent in attrs: + attrs[branch_parent] = parent_units.convert( + attrs[branch_parent], cube_coord.units) + + branch_child = 'branch_time_in_child' + if branch_child in attrs: + attrs[branch_child] = old_units.convert( + attrs[branch_child], cube_coord.units) + + def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: + """Fix time bounds.""" + times = {'time', 'time1', 'time2', 'time3'} + key = times.intersection(self.vardef.coordinates) + cmor = self.vardef.coordinates[' '.join(key)] + if cmor.must_have_bounds == 'yes' and not cube_coord.has_bounds(): + cube_coord.bounds = get_time_bounds(cube_coord, self.frequency) + self._warning_msg( + cube, + "Added guessed bounds to coordinate %s", + cube_coord.var_name, + ) + + def _fix_time_coord(self, cube: Cube) -> Cube: + """Fix time coordinate.""" + # Make sure to get dimensional time coordinate if possible + if cube.coords('time', dim_coords=True): + cube_coord = cube.coord('time', dim_coords=True) + elif cube.coords('time'): + cube_coord = cube.coord('time') + else: + return cube + + # Cannot fix wrong time that are not references + if not cube_coord.units.is_time_reference(): + return cube + + # Fix time units + self._fix_time_units(cube, cube_coord) + + # Remove time_origin from coordinate attributes + cube_coord.attributes.pop('time_origin', None) + + # Fix time bounds + self._fix_time_bounds(cube, cube_coord) + + return cube + + def _fix_coord( + self, + cube: Cube, + cmor_coord: CoordinateInfo, + cube_coord: Coord, + ) -> tuple[Cube, Coord]: + """Fix non-time coordinate.""" + self._fix_coord_units(cube, cmor_coord, cube_coord) + (cube, cube_coord) = self._fix_longitude_0_360( + cube, cmor_coord, cube_coord + ) + self._fix_coord_bounds(cube, cmor_coord, cube_coord) + (cube, cube_coord) = self._fix_coord_direction( + cube, cmor_coord, cube_coord + ) + self._fix_requested_coord_values(cube, cmor_coord, cube_coord) + return (cube, cube_coord) + + def _fix_coords(self, cube: Cube) -> Cube: + """Fix non-time coordinates.""" + for cmor_coord in self.vardef.coordinates.values(): + + # Cannot fix generic level coords with no unique CMOR information + if cmor_coord.generic_level and not cmor_coord.out_name: + continue + + # Try to get coordinate from cube; if it does not exists, skip + if not cube.coords(var_name=cmor_coord.out_name): + continue + cube_coord = cube.coord(var_name=cmor_coord.out_name) + + # Fixes for time coord are done separately + if cube_coord.var_name == 'time': + continue + + # Fixes + (cube, cube_coord) = self._fix_coord(cube, cmor_coord, cube_coord) + + return cube diff --git a/esmvalcore/cmor/_utils.py b/esmvalcore/cmor/_utils.py new file mode 100644 index 0000000000..4b083f19de --- /dev/null +++ b/esmvalcore/cmor/_utils.py @@ -0,0 +1,219 @@ +"""Utilities for CMOR module.""" +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import Optional + +from iris.coords import Coord +from iris.cube import Cube +from iris.exceptions import CoordinateNotFoundError + +from esmvalcore.cmor.table import CMOR_TABLES, CoordinateInfo, VariableInfo + +logger = logging.getLogger(__name__) + +_ALTERNATIVE_GENERIC_LEV_COORDS = { + 'alevel': { + 'CMIP5': ['alt40', 'plevs'], + 'CMIP6': ['alt16', 'plev3'], + 'obs4MIPs': ['alt16', 'plev3'], + }, + 'zlevel': { + 'CMIP3': ['pressure'], + }, +} + + +def _get_alternative_generic_lev_coord( + cube: Cube, + coord_name: str, + cmor_table_type: str, +) -> tuple[CoordinateInfo, Coord]: + """Find alternative generic level coordinate in cube. + + Parameters + ---------- + cube: + Cube to be checked. + coord_name: + Name of the generic level coordinate. + cmor_table_type: + CMOR table type, e.g., CMIP3, CMIP5, CMIP6. Note: This is NOT the + project of the dataset, but rather the entry `cmor_type` in + `config-developer.yml`. + + Returns + ------- + tuple[CoordinateInfo, Coord] + Coordinate information from the CMOR tables and the corresponding + coordinate in the cube. + + Raises + ------ + ValueError + No valid alternative generic level coordinate present in cube. + + """ + alternatives_for_coord = _ALTERNATIVE_GENERIC_LEV_COORDS.get( + coord_name, {} + ) + allowed_alternatives = alternatives_for_coord.get(cmor_table_type, []) + + # Check if any of the allowed alternative coordinates is present in the + # cube + for allowed_alternative in allowed_alternatives: + cmor_coord = CMOR_TABLES[cmor_table_type].coords[allowed_alternative] + if cube.coords(var_name=cmor_coord.out_name): + cube_coord = cube.coord(var_name=cmor_coord.out_name) + return (cmor_coord, cube_coord) + + raise ValueError( + f"Found no valid alternative coordinate for generic level coordinate " + f"'{coord_name}'" + ) + + +def _get_generic_lev_coord_names( + cube: Cube, + cmor_coord: CoordinateInfo, +) -> tuple[str | None, str | None, str | None]: + """Try to get names of a generic level coordinate. + + Parameters + ---------- + cube: + Cube to be checked. + cmor_coord: + Coordinate information from the CMOR table with a non-emmpty + `generic_lev_coords` :obj:`dict`. + + Returns + ------- + tuple[str | None, str | None, str | None] + Tuple of `standard_name`, `out_name`, and `name` of the generic level + coordinate present in the cube. Values are ``None`` if generic level + coordinate has not been found in cube. + + """ + standard_name = None + out_name = None + name = None + + # Iterate over all possible generic level coordinates + for coord in cmor_coord.generic_lev_coords.values(): + # First, try to use var_name to find coordinate + if cube.coords(var_name=coord.out_name): + cube_coord = cube.coord(var_name=coord.out_name) + out_name = coord.out_name + if cube_coord.standard_name == coord.standard_name: + standard_name = coord.standard_name + name = coord.name + + # Second, try to use standard_name to find coordinate + elif cube.coords(coord.standard_name): + standard_name = coord.standard_name + name = coord.name + + return (standard_name, out_name, name) + + +def _get_new_generic_level_coord( + var_info: VariableInfo, + generic_level_coord: CoordinateInfo, + generic_level_coord_name: str, + new_coord_name: str | None, +) -> CoordinateInfo: + """Get new generic level coordinate. + + There are a variety of possible options for each generic level coordinate + (e.g., `alevel`) which is actually present in a cube, for example, + `hybrid_height` or `standard_hybrid_sigma`. This function returns the new + coordinate (e.g., `new_coord_name=hybrid_height`) with the relevant + metadata. + + Note + ---- + This alters the corresponding entry of the original generic level + coordinate's `generic_level_coords` attribute (i.e., + ``generic_level_coord.generic_level_coords[new_coord_name]`) in-place! + + Parameters + ---------- + var_info: + CMOR variable information. + generic_level_coord: + Original generic level coordinate. + generic_level_coord_name: + Original name of the generic level coordinate (e.g., `alevel`). + new_coord_name: + Name of the new generic level coordinate (e.g., `hybrid_height`). + + Returns + ------- + CoordinateInfo + New generic level coordinate. + + """ + new_coord = generic_level_coord.generic_lev_coords[new_coord_name] + new_coord.generic_level = True + new_coord.generic_lev_coords = ( + var_info.coordinates[generic_level_coord_name].generic_lev_coords + ) + return new_coord + + +def _get_simplified_calendar(calendar: str) -> str: + """Simplify calendar.""" + calendar_aliases = { + 'all_leap': '366_day', + 'noleap': '365_day', + 'gregorian': 'standard', + } + return calendar_aliases.get(calendar, calendar) + + +def _is_unstructured_grid(cube: Cube) -> bool: + """Check if cube uses unstructured grid.""" + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + pass + else: + if lat.ndim == 1 and (cube.coord_dims(lat) == cube.coord_dims(lon)): + return True + return False + + +def _get_single_cube( + cube_list: Sequence[Cube], + short_name: str, + dataset_str: Optional[str] = None, +) -> Cube: + if len(cube_list) == 1: + return cube_list[0] + cube = None + for raw_cube in cube_list: + if raw_cube.var_name == short_name: + cube = raw_cube + break + + if dataset_str is None: + dataset_str = '' + else: + dataset_str = f' in {dataset_str}' + + if not cube: + raise ValueError( + f"More than one cube found for variable {short_name}{dataset_str} " + f"but none of their var_names match the expected.\nFull list of " + f"cubes encountered: {cube_list}" + ) + logger.warning( + "Found variable %s%s, but there were other present in the file. Those " + "extra variables are usually metadata (cell area, latitude " + "descriptions) that was not saved according to CF-conventions. It is " + "possible that errors appear further on because of this.\nFull list " + "of cubes encountered: %s", short_name, dataset_str, cube_list) + return cube diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 4eb057730b..43214168b8 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -1,7 +1,12 @@ """Module for checking iris cubes against their CMOR definitions.""" +from __future__ import annotations + import logging -from datetime import datetime +import warnings +from collections.abc import Callable from enum import IntEnum +from functools import cached_property +from typing import Optional import cf_units import iris.coord_categorisation @@ -9,10 +14,18 @@ import iris.exceptions import iris.util import numpy as np +from iris.cube import Cube -from esmvalcore.iris_helpers import date2num - -from .table import CMOR_TABLES +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor._utils import ( + _get_alternative_generic_lev_coord, + _get_generic_lev_coord_names, + _get_new_generic_level_coord, + _get_simplified_calendar, + _is_unstructured_grid, +) +from esmvalcore.cmor.table import get_var_info +from esmvalcore.exceptions import ESMValCoreDeprecationWarning class CheckLevels(IntEnum): @@ -34,49 +47,6 @@ class CheckLevels(IntEnum): """Do not fail for any discrepancy with CMOR standards.""" -def _get_next_month(month, year): - if month != 12: - return month + 1, year - return 1, year + 1 - - -def _get_time_bounds(time, freq): - bounds = [] - dates = time.units.num2date(time.points) - for step, date in enumerate(dates): - month = date.month - year = date.year - if freq in ['mon', 'mo']: - next_month, next_year = _get_next_month(month, year) - min_bound = date2num(datetime(year, month, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), - time.units, time.dtype) - elif freq == 'yr': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), - time.units, time.dtype) - elif freq == 'dec': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), - time.units, time.dtype) - else: - delta = { - 'day': 12 / 24, - '6hr': 3 / 24, - '3hr': 1.5 / 24, - '1hr': 0.5 / 24, - } - point = time.points[step] - min_bound = point - delta[freq] - max_bound = point + delta[freq] - bounds.append([min_bound, max_bound]) - - return np.array(bounds) - - class CMORCheckError(Exception): """Exception raised when a cube does not pass the CMORCheck.""" @@ -84,9 +54,6 @@ class CMORCheckError(Exception): class CMORCheck(): """Class used to check the CMOR-compliance of the data. - It can also fix some minor errors and does some minor data - homogeneization: - Parameters ---------- cube: iris.cube.Cube: @@ -94,13 +61,23 @@ class CMORCheck(): var_info: variables_info.VariableInfo Variable info to check. frequency: str - Expected frequency for the data. + Expected frequency for the data. If not given, use the one from the + variable information. fail_on_error: bool If true, CMORCheck stops on the first error. If false, it collects all possible errors before stopping. automatic_fixes: bool If True, CMORCheck will try to apply automatic fixes for any detected error, if possible. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.fix_metadata`, + :func:`~esmvalcore.preprocessor.fix_data`, or + :meth:`esmvalcore.dataset.Dataset.load` (which automatically + includes the first two functions) instead. Fixes and CMOR checks + have been clearly separated in ESMValCore version 2.10.0. check_level: CheckLevels Level of strictness of the checks. @@ -131,7 +108,6 @@ def __init__(self, self._errors = list() self._warnings = list() self._debug_messages = list() - self._unstructured = None self._cmor_var = var_info if not frequency: @@ -139,47 +115,61 @@ def __init__(self, self.frequency = frequency self.automatic_fixes = automatic_fixes - def _is_unstructured_grid(self): - if self._unstructured is None: - self._unstructured = False - try: - lat = self._cube.coord('latitude') - lon = self._cube.coord('longitude') - except iris.exceptions.CoordinateNotFoundError: - pass - else: - if lat.ndim == 1 and (self._cube.coord_dims(lat) - == self._cube.coord_dims(lon)): - self._unstructured = True - return self._unstructured - - def check_metadata(self, logger=None): + # Deprecate automatic_fixes (remove in v2.12) + if automatic_fixes: + msg = ( + "The option `automatic_fixes` has been deprecated in " + "ESMValCore version 2.10.0 and is scheduled for removal in " + "version 2.12.0. Please use the functions " + "esmvalcore.preprocessor.fix_metadata(), " + "esmvalcore.preprocessor.fix_data(), or " + "esmvalcore.dataset.Dataset.load() (which automatically " + "includes the first two functions) instead. Fixes and CMOR " + "checks have been clearly separated in ESMValCore version " + "2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + + # TODO: remove in v2.12 + + self._generic_fix = GenericFix(var_info, frequency=frequency) + + @cached_property + def _unstructured_grid(self) -> bool: + """Cube uses unstructured grid.""" + return _is_unstructured_grid(self._cube) + + def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube metadata. - Perform all the tests that do not require to have the data in memory. - - It will also report some warnings in case of minor errors and - homogenize some data: - - - Equivalent calendars will all default to the same name. - - Time units will be set to days since 1850-01-01 + It will also report some warnings in case of minor errors. Parameters ---------- - logger: logging.Logger + logger: Given logger. + Returns + ------- + iris.cube.Cube + Checked cube. + Raises ------ CMORCheckError If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. + """ if logger is not None: self._logger = logger + # TODO: remove in v2.12 + if self.automatic_fixes: + [self._cube] = self._generic_fix.fix_metadata([self._cube]) + self._check_var_metadata() self._check_fill_value() self._check_multiple_coords_same_stdname() @@ -196,10 +186,9 @@ def check_metadata(self, logger=None): return self._cube - def check_data(self, logger=None): + def check_data(self, logger: Optional[logging.Logger] = None) -> Cube: """Check the cube data. - Performs all the tests that require to have the data in memory. Assumes that metadata is correct, so you must call check_metadata prior to this. @@ -207,28 +196,35 @@ def check_data(self, logger=None): Parameters ---------- - logger: logging.Logger + logger: Given logger. + Returns + ------- + iris.cube.Cube + Checked cube. + Raises ------ CMORCheckError If errors are found. If fail_on_error attribute is set to True, raises as soon as an error is detected. If set to False, it perform all checks and then raises. + """ if logger is not None: self._logger = logger - if self._cmor_var.units: - units = self._get_effective_units() - if str(self._cube.units) != units: - self._cube.convert_units(units) + # TODO: remove in v2.12 + if self.automatic_fixes: + self._cube = self._generic_fix.fix_data(self._cube) self._check_coords_data() + self.report_debug_messages() self.report_warnings() self.report_errors() + return self._cube def report_errors(self): @@ -251,13 +247,7 @@ def report_errors(self): raise CMORCheckError(msg) def report_warnings(self): - """Report detected warnings to the given logger. - - Parameters - ---------- - logger: logging.Logger - Given logger - """ + """Report detected warnings to the given logger.""" if self.has_warnings(): msg = '\n'.join([ f'There were warnings in variable {self._cube.var_name}:', @@ -268,13 +258,7 @@ def report_warnings(self): self._logger.warning(msg) def report_debug_messages(self): - """Report detected debug messages to the given logger. - - Parameters - ---------- - logger: logging.Logger - Given logger. - """ + """Report detected debug messages to the given logger.""" if self.has_debug_messages(): msg = '\n'.join([ f'There were metadata changes in variable ' @@ -298,49 +282,24 @@ def _check_var_metadata(self): # Check standard_name if self._cmor_var.standard_name: if self._cube.standard_name != self._cmor_var.standard_name: - if self.automatic_fixes: - self.report_warning( - 'Standard name for {} changed from {} to {}', - self._cube.var_name, self._cube.standard_name, - self._cmor_var.standard_name) - self._cube.standard_name = self._cmor_var.standard_name - else: - self.report_error(self._attr_msg, self._cube.var_name, - 'standard_name', - self._cmor_var.standard_name, - self._cube.standard_name) + self.report_error(self._attr_msg, self._cube.var_name, + 'standard_name', + self._cmor_var.standard_name, + self._cube.standard_name) # Check long_name if self._cmor_var.long_name: if self._cube.long_name != self._cmor_var.long_name: - if self.automatic_fixes: - self.report_warning( - 'Long name for {} changed from {} to {}', - self._cube.var_name, self._cube.long_name, - self._cmor_var.long_name) - self._cube.long_name = self._cmor_var.long_name - else: - self.report_error(self._attr_msg, self._cube.var_name, - 'long_name', self._cmor_var.long_name, - self._cube.long_name) + self.report_error(self._attr_msg, self._cube.var_name, + 'long_name', self._cmor_var.long_name, + self._cube.long_name) # Check units - if (self.automatic_fixes and self._cube.attributes.get( - 'invalid_units', '').lower() == 'psu'): - self._cube.units = '1.0' - del self._cube.attributes['invalid_units'] - if self._cmor_var.units: units = self._get_effective_units() if self._cube.units != units: - if not self._cube.units.is_convertible(units): - self.report_error(f'Variable {self._cube.var_name} units ' - f'{self._cube.units} can not be ' - f'converted to {self._cmor_var.units}') - else: - self.report_warning( - f'Variable {self._cube.var_name} units ' - f'{self._cube.units} will be ' - f'converted to {self._cmor_var.units}') + self.report_error(self._attr_msg, self._cube.var_name, + 'units', self._cmor_var.units, + self._cube.units) # Check other variable attributes that match entries in cube.attributes attrs = ('positive', ) @@ -357,6 +316,7 @@ def _check_var_metadata(self): def _get_effective_units(self): """Get effective units.""" + # TODO: remove entire function in v2.12 if self._cmor_var.units.lower() == 'psu': units = '1.0' else: @@ -423,20 +383,7 @@ def _check_dim_names(self): except iris.exceptions.CoordinateNotFoundError: try: coord = self._cube.coord(coordinate.standard_name) - if self._cmor_var.table_type in 'CMIP6' and \ - coord.ndim > 1 and \ - coord.standard_name in ['latitude', 'longitude']: - self.report_debug_message( - 'Multidimensional {0} coordinate is not set ' - 'in CMOR standard. ESMValTool will change ' - 'the original value of {1} to {2} to match ' - 'the one-dimensional case.', - coordinate.standard_name, - coord.var_name, - coordinate.out_name, - ) - coord.var_name = coordinate.out_name - elif coord.standard_name in ['region', 'area_type']: + if coord.standard_name in ['region', 'area_type']: self.report_debug_message( 'Coordinate {0} has var name {1} ' 'instead of {2}. ' @@ -467,33 +414,17 @@ def _check_dim_names(self): def _check_generic_level_dim_names(self, key, coordinate): """Check name of generic level coordinate.""" - standard_name = None - out_name = None - name = None if coordinate.generic_lev_coords: - for coord in coordinate.generic_lev_coords.values(): - try: - cube_coord = self._cube.coord(var_name=coord.out_name) - out_name = coord.out_name - if cube_coord.standard_name == coord.standard_name: - standard_name = coord.standard_name - name = coord.name - except iris.exceptions.CoordinateNotFoundError: - try: - cube_coord = self._cube.coord( - var_name=coord.standard_name) - standard_name = coord.standard_name - name = coord.name - except iris.exceptions.CoordinateNotFoundError: - pass + (standard_name, out_name, name) = _get_generic_lev_coord_names( + self._cube, coordinate + ) if standard_name: if not out_name: self.report_error( f'Generic level coordinate {key} has wrong var_name.') - level = coordinate.generic_lev_coords[name] - level.generic_level = True - level.generic_lev_coords = self._cmor_var.coordinates[ - key].generic_lev_coords + level = _get_new_generic_level_coord( + self._cmor_var, coordinate, key, name + ) self._cmor_var.coordinates[key] = level self.report_debug_message(f'Generic level coordinate {key} ' 'will be checked against ' @@ -506,17 +437,6 @@ def _check_generic_level_dim_names(self, key, coordinate): else: self._check_alternative_dim_names(key) - ALTERNATIVE_GENERIC_LEV_COORDS = { - 'alevel': { - 'CMIP5': ['alt40', 'plevs'], - 'CMIP6': ['alt16', 'plev3'], - 'obs4MIPs': ['alt16', 'plev3'], - }, - 'zlevel': { - 'CMIP3': ['pressure'], - }, - } - def _check_alternative_dim_names(self, key): """Check for viable alternatives to generic level coordinates. @@ -549,42 +469,34 @@ def _check_alternative_dim_names(self, key): For ``cmor_strict=False`` project (like OBS) the check for requested values might be disabled. """ - table_type = self._cmor_var.table_type - alternative_coord = None - allowed_alternatives = self.ALTERNATIVE_GENERIC_LEV_COORDS.get( - key, {}).get(table_type, []) - - # Check if any of the allowed alternative coordinates is present in the - # cube - for allowed_alternative in allowed_alternatives: - coord_info = CMOR_TABLES[table_type].coords[allowed_alternative] - try: - cube_coord = self._cube.coord(var_name=coord_info.out_name) - except iris.exceptions.CoordinateNotFoundError: - pass - else: - if cube_coord.standard_name == coord_info.standard_name: - alternative_coord = coord_info - break - self.report_error( - f"Found alternative coordinate '{coord_info.out_name}' " - f"for generic level coordinate '{key}' with wrong " - f"standard_name '{cube_coord.standard_name}' (expected " - f"'{coord_info.standard_name}')") - break + try: + (alternative_coord, + cube_coord) = _get_alternative_generic_lev_coord( + self._cube, key, self._cmor_var.table_type + ) # No valid alternative coordinate found -> critical error - if alternative_coord is None: + except ValueError: self.report_critical(self._does_msg, key, 'exist') return + # Wrong standard_name -> error + if cube_coord.standard_name != alternative_coord.standard_name: + self.report_error( + f"Found alternative coordinate '{alternative_coord.out_name}' " + f"for generic level coordinate '{key}' with wrong " + f"standard_name {cube_coord.standard_name}' (expected " + f"'{alternative_coord.standard_name}')" + ) + return + # Valid alternative coordinate found -> perform checks on it self.report_warning( f"Found alternative coordinate '{alternative_coord.out_name}' " f"for generic level coordinate '{key}'. Subsequent warnings about " f"levels that are not contained in '{alternative_coord.out_name}' " f"can be safely ignored.") - self._check_coord(alternative_coord, cube_coord, self._cube.var_name) + self._check_coord(alternative_coord, cube_coord, cube_coord.var_name) def _check_coords(self): """Check coordinates.""" @@ -616,6 +528,12 @@ def _check_coords_data(self): except iris.exceptions.CoordinateNotFoundError: continue + # TODO: remove in v2.12 + if self.automatic_fixes: + (self._cube, coord) = self._generic_fix._fix_coord_direction( + self._cube, coordinate, coord + ) + self._check_coord_monotonicity_and_direction( coordinate, coord, var_name) @@ -625,57 +543,24 @@ def _check_coord(self, cmor, coord, var_name): return if cmor.units: if str(coord.units) != cmor.units: - fixed = False - if self.automatic_fixes: - try: - old_unit = coord.units - new_unit = cf_units.Unit(cmor.units, - coord.units.calendar) - coord.convert_units(new_unit) - fixed = True - self.report_warning( - f'Coordinate {coord.var_name} units ' - f'{str(old_unit)} ' - f'converted to {cmor.units}') - except ValueError: - pass - if not fixed: - self.report_critical(self._attr_msg, var_name, 'units', - cmor.units, coord.units) + self.report_critical(self._attr_msg, var_name, 'units', + cmor.units, coord.units) self._check_coord_points(cmor, coord, var_name) def _check_coord_bounds(self, cmor, coord, var_name): if cmor.must_have_bounds == 'yes' and not coord.has_bounds(): - if self.automatic_fixes: - try: - coord.guess_bounds() - except ValueError as ex: - self.report_warning( - 'Can not guess bounds for coordinate {0} ' - 'from var {1}: {2}', coord.var_name, var_name, ex) - else: - self.report_warning( - 'Added guessed bounds to coordinate {0} from var {1}', - coord.var_name, var_name) - else: - self.report_warning( - 'Coordinate {0} from var {1} does not have bounds', - coord.var_name, var_name) + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + coord.var_name, var_name) - def _check_time_bounds(self, freq, time): + def _check_time_bounds(self, time): times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self._cmor_var.coordinates) cmor = self._cmor_var.coordinates[" ".join(key)] if cmor.must_have_bounds == 'yes' and not time.has_bounds(): - if self.automatic_fixes: - time.bounds = _get_time_bounds(time, freq) - self.report_warning( - 'Added guessed bounds to coordinate {0} from var {1}', - time.var_name, self._cmor_var.short_name) - else: - self.report_warning( - 'Coordinate {0} from var {1} does not have bounds', - time.var_name, self._cmor_var.short_name) + self.report_warning( + 'Coordinate {0} from var {1} does not have bounds', + time.var_name, self._cmor_var.short_name) def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): """Check monotonicity and direction of coordinate.""" @@ -684,8 +569,8 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): if coord.dtype.kind == 'U': return - if self._is_unstructured_grid() and \ - coord.standard_name in ['latitude', 'longitude']: + if (self._unstructured_grid and + coord.standard_name in ['latitude', 'longitude']): self.report_debug_message( f'Coordinate {coord.standard_name} appears to belong to ' 'an unstructured grid. Skipping monotonicity and ' @@ -694,118 +579,42 @@ def _check_coord_monotonicity_and_direction(self, cmor, coord, var_name): if not coord.is_monotonic(): self.report_critical(self._is_msg, var_name, 'monotonic') + if len(coord.core_points()) == 1: return + if cmor.stored_direction: if cmor.stored_direction == 'increasing': if coord.core_points()[0] > coord.core_points()[1]: - if not self.automatic_fixes or coord.ndim > 1: - self.report_critical(self._is_msg, var_name, - 'increasing') - else: - self._reverse_coord(coord) + self.report_critical(self._is_msg, var_name, 'increasing') elif cmor.stored_direction == 'decreasing': if coord.core_points()[0] < coord.core_points()[1]: - if not self.automatic_fixes or coord.ndim > 1: - self.report_critical(self._is_msg, var_name, - 'decreasing') - else: - self._reverse_coord(coord) - - def _reverse_coord(self, coord): - """Reverse coordinate.""" - if coord.ndim == 1: - self._cube = iris.util.reverse(self._cube, - self._cube.coord_dims(coord)) - reversed_coord = self._cube.coord(var_name=coord.var_name) - if reversed_coord.has_bounds(): - bounds = reversed_coord.bounds - right_bounds = bounds[:-2, 1] - left_bounds = bounds[1:-1, 0] - if np.all(right_bounds != left_bounds): - reversed_coord.bounds = np.fliplr(bounds) - coord = reversed_coord - self.report_debug_message(f'Coordinate {coord.var_name} values' - 'have been reversed.') + self.report_critical(self._is_msg, var_name, 'decreasing') def _check_coord_points(self, coord_info, coord, var_name): """Check coordinate points: values, bounds and monotonicity.""" # Check requested coordinate values exist in coord.points self._check_requested_values(coord, coord_info, var_name) - l_fix_coord_value = False - # Check coordinate value ranges if coord_info.valid_min: valid_min = float(coord_info.valid_min) if np.any(coord.core_points() < valid_min): - if coord_info.standard_name == 'longitude' and \ - self.automatic_fixes: - l_fix_coord_value = self._check_longitude_min( - coord, var_name) - else: - self.report_critical(self._vals_msg, var_name, - '< {} ='.format('valid_min'), - valid_min) + self.report_critical(self._vals_msg, var_name, + '< {} ='.format('valid_min'), + valid_min) if coord_info.valid_max: valid_max = float(coord_info.valid_max) if np.any(coord.core_points() > valid_max): - if coord_info.standard_name == 'longitude' and \ - self.automatic_fixes: - l_fix_coord_value = self._check_longitude_max( - coord, var_name) - else: - self.report_critical(self._vals_msg, var_name, - '> {} ='.format('valid_max'), - valid_max) - - if l_fix_coord_value: - # cube.intersection only works for cells with 0 or 2 bounds - # Note: nbounds==0 means there are no bounds given, nbounds==2 - # implies a regular grid with bounds in the grid direction, - # nbounds>2 implies an irregular grid with bounds given as vertices - # of the cell polygon. - if coord.ndim == 1 and coord.nbounds in (0, 2): - lon_extent = iris.coords.CoordExtent(coord, 0.0, 360., True, - False) - self._cube = self._cube.intersection(lon_extent) - else: - new_lons = coord.core_points().copy() - new_lons = self._set_range_in_0_360(new_lons) - if coord.core_bounds() is not None: - new_bounds = coord.core_bounds().copy() - new_bounds = self._set_range_in_0_360(new_bounds) - else: - new_bounds = None - new_coord = coord.copy(new_lons, new_bounds) - dims = self._cube.coord_dims(coord) - self._cube.remove_coord(coord) - self._cube.add_aux_coord(new_coord, dims) - coord = self._cube.coord(var_name=var_name) + self.report_critical(self._vals_msg, var_name, + '> {} ='.format('valid_max'), + valid_max) + self._check_coord_bounds(coord_info, coord, var_name) self._check_coord_monotonicity_and_direction(coord_info, coord, var_name) - def _check_longitude_max(self, coord, var_name): - if np.any(coord.core_points() > 720): - self.report_critical( - f'{var_name} longitude coordinate has values > 720 degrees') - return False - return True - - def _check_longitude_min(self, coord, var_name): - if np.any(coord.core_points() < -360): - self.report_critical( - f'{var_name} longitude coordinate has values < -360 degrees') - return False - return True - - @staticmethod - def _set_range_in_0_360(array): - """Convert longitude coordinate to [0, 360].""" - return (array + 360.0) % 360.0 - def _check_requested_values(self, coord, coord_info, var_name): """Check requested values.""" if coord_info.requested: @@ -818,17 +627,6 @@ def _check_requested_values(self, coord, coord_info, var_name): cmor_points = np.array(coord_info.requested, dtype=float) except ValueError: cmor_points = coord_info.requested - else: - atol = 1e-7 * np.mean(cmor_points) - if (self.automatic_fixes - and coord.core_points().shape == cmor_points.shape - and np.allclose( - coord.core_points(), - cmor_points, - rtol=1e-7, - atol=atol, - )): - coord.points = cmor_points for point in cmor_points: if point not in coord.core_points(): self.report_warning(self._contain_msg, var_name, @@ -853,12 +651,7 @@ def _check_time_coord(self): self.report_critical(self._does_msg, var_name, 'have time reference units') else: - old_units = coord.units - coord.convert_units( - cf_units.Unit('days since 1850-1-1 00:00:00', - calendar=coord.units.calendar)) - simplified_cal = self._simplify_calendar(coord.units.calendar) - coord.units = cf_units.Unit(coord.units.origin, simplified_cal) + simplified_cal = _get_simplified_calendar(coord.units.calendar) attrs = self._cube.attributes parent_time = 'parent_time_units' if parent_time in attrs: @@ -866,8 +659,7 @@ def _check_time_coord(self): pass else: try: - parent_units = cf_units.Unit(attrs[parent_time], - simplified_cal) + cf_units.Unit(attrs[parent_time], simplified_cal) except ValueError: self.report_warning('Attribute parent_time_units has ' 'a wrong format and cannot be ' @@ -875,17 +667,8 @@ def _check_time_coord(self): 'be added to convert properly ' 'attributes branch_time_in_parent ' 'and branch_time_in_child.') - else: - attrs[parent_time] = 'days since 1850-1-1 00:00:00' - branch_parent = 'branch_time_in_parent' - if branch_parent in attrs: - attrs[branch_parent] = parent_units.convert( - attrs[branch_parent], coord.units) - branch_child = 'branch_time_in_child' - if branch_child in attrs: - attrs[branch_child] = old_units.convert( - attrs[branch_child], coord.units) + # Check frequency tol = 0.001 intervals = {'dec': (3600, 3660), 'day': (1, 1)} freq = self.frequency @@ -941,18 +724,8 @@ def _check_time_coord(self): msg = '{}: Frequency {} does not match input data' self.report_error(msg, var_name, freq) break - self._check_time_bounds(freq, coord) - # remove time_origin from attributes - coord.attributes.pop('time_origin', None) - - @staticmethod - def _simplify_calendar(calendar): - calendar_aliases = { - 'all_leap': '366_day', - 'noleap': '365_day', - 'gregorian': 'standard', - } - return calendar_aliases.get(calendar, calendar) + + self._check_time_bounds(coord) def has_errors(self): """Check if there are reported errors. @@ -1071,30 +844,19 @@ def report_debug_message(self, message, *args): self.report(CheckLevels.DEBUG, message, *args) -def _get_cmor_checker(table, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=CheckLevels.DEFAULT, - automatic_fixes=False): - """Get a CMOR checker/fixer.""" - if table not in CMOR_TABLES: - raise NotImplementedError( - "No CMOR checker implemented for table {}." - "\nThe following options are available: {}".format( - table, ', '.join(CMOR_TABLES))) - - cmor_table = CMOR_TABLES[table] - if table == 'CORDEX' and mip.endswith('hr'): - # CORDEX X-hourly tables define the mip - # as ending in 'h' instead of 'hr'. - mip = mip.replace('hr', 'h') - var_info = cmor_table.get_variable(mip, short_name) - if var_info is None: - var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) - - def _checker(cube): +def _get_cmor_checker( + project: str, + mip: str, + short_name: str, + frequency: None | str = None, + fail_on_error: bool = False, + check_level: CheckLevels = CheckLevels.DEFAULT, + automatic_fixes: bool = False, # TODO: remove in v2.12 +) -> Callable[[Cube], CMORCheck]: + """Get a CMOR checker.""" + var_info = get_var_info(project, mip, short_name) + + def _checker(cube: Cube) -> CMORCheck: return CMORCheck(cube, var_info, frequency=frequency, @@ -1105,105 +867,143 @@ def _checker(cube): return _checker -def cmor_check_metadata(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=CheckLevels.DEFAULT): +def cmor_check_metadata( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if metadata conforms to variable's CMOR definition. None of the checks at this step will force the cube to load the data. Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name. - frequency: str - Data frequency. - check_level: CheckLevels + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - checker = _get_cmor_checker(cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - checker(cube).check_metadata() + checker = _get_cmor_checker( + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = checker(cube).check_metadata() return cube -def cmor_check_data(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=CheckLevels.DEFAULT): +def cmor_check_data( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if data conforms to variable's CMOR definition. - The checks performed at this step require the data in memory. - Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name - frequency: str - Data frequency - check_level: CheckLevels + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - checker = _get_cmor_checker(cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - checker(cube).check_data() + checker = _get_cmor_checker( + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = checker(cube).check_data() return cube -def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): +def cmor_check( + cube: Cube, + cmor_table: str, + mip: str, + short_name: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, +) -> Cube: """Check if cube conforms to variable's CMOR definition. - Equivalent to calling cmor_check_metadata and cmor_check_data - consecutively. + Equivalent to calling :func:`cmor_check_metadata` and + :func:`cmor_check_data` consecutively. Parameters ---------- - cube: iris.cube.Cube + cube: Data cube to check. - cmor_table: str - CMOR definitions to use. + cmor_table: + CMOR definitions to use (i.e., the variable's project). mip: - Variable's mip. - short_name: str + Variable's MIP. + short_name: Variable's short name. - frequency: str - Data frequency. - check_level: enum.IntEnum + frequency: + Data frequency. If not given, use the one from the CMOR table of the + variable. + check_level: Level of strictness of the checks. + + Returns + ------- + iris.cube.Cube + Checked cube. + """ - cmor_check_metadata(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=check_level) - cmor_check_data(cube, - cmor_table, - mip, - short_name, - frequency, - check_level=check_level) + cube = cmor_check_metadata( + cube, + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) + cube = cmor_check_data( + cube, + cmor_table, + mip, + short_name, + frequency=frequency, + check_level=check_level, + ) return cube diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 1034435f92..d05af5ef64 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -7,14 +7,17 @@ from __future__ import annotations import logging +import warnings from collections import defaultdict +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING, Optional -from iris.cube import CubeList +from iris.cube import Cube, CubeList -from ._fixes.fix import Fix -from .check import CheckLevels, _get_cmor_checker +from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor.check import CheckLevels, _get_cmor_checker +from esmvalcore.exceptions import ESMValCoreDeprecationWarning if TYPE_CHECKING: from ..config import Session @@ -31,11 +34,12 @@ def fix_file( output_dir: Path, add_unique_suffix: bool = False, session: Optional[Session] = None, + frequency: Optional[str] = None, **extra_facets, ) -> str | Path: """Fix files before ESMValTool can load them. - This fixes are only for issues that prevent iris from loading the cube or + These fixes are only for issues that prevent iris from loading the cube or that cannot be fixed after the cube is loaded. Original files are not overwritten. @@ -58,6 +62,8 @@ def fix_file( Adds a unique suffix to `output_dir` for thread safety. session: Current session which includes configuration and directory information. + frequency: + Variable's data frequency, if available. **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. @@ -75,6 +81,7 @@ def fix_file( 'project': project, 'dataset': dataset, 'mip': mip, + 'frequency': frequency, }) for fix in Fix.get_fixes(project=project, @@ -82,61 +89,81 @@ def fix_file( mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session): + session=session, + frequency=frequency): file = fix.fix_file( file, output_dir, add_unique_suffix=add_unique_suffix ) return file -def fix_metadata(cubes, - short_name, - project, - dataset, - mip, - frequency=None, - check_level=CheckLevels.DEFAULT, - session: Optional[Session] = None, - **extra_facets): - """Fix cube metadata if fixes are required and check it anyway. +def fix_metadata( + cubes: Sequence[Cube], + short_name: str, + project: str, + dataset: str, + mip: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, + session: Optional[Session] = None, + **extra_facets, +) -> CubeList: + """Fix cube metadata if fixes are required. - This method collects all the relevant fixes for a given variable, applies - them and checks the resulting cube (or the original if no fixes were - needed) metadata to ensure that it complies with the standards of its - project CMOR tables. + This method collects all the relevant fixes (including generic ones) for a + given variable and applies them. Parameters ---------- - cubes: iris.cube.CubeList + cubes: Cubes to fix. - short_name: str + short_name: Variable's short name. - project: str + project: Project of the dataset. - dataset: str + dataset: Name of the dataset. - mip: str + mip: Variable's MIP. - frequency: str, optional + frequency: Variable's data frequency, if available. - check_level: CheckLevels - Level of strictness of the checks. Set to default. - session: Session, optional + check_level: + Level of strictness of the checks. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.cmor_check_metadata`, + :func:`~esmvalcore.preprocessor.cmor_check_data`, or + :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function + will no longer perform CMOR checks. Fixes and CMOR checks have been + clearly separated in ESMValCore version 2.10.0. + session: Current session which includes configuration and directory information. - **extra_facets: dict, optional + **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. Returns ------- - iris.cube.Cube: - Fixed and checked cube. + iris.cube.CubeList + Fixed cubes. - Raises - ------ - CMORCheckError - If the checker detects errors in the metadata that it can not fix. """ + # Deprecate CMOR checks (remove in v2.12) + if check_level != CheckLevels.DEFAULT: + msg = ( + "The option `check_level` has been deprecated in ESMValCore " + "version 2.10.0 and is scheduled for removal in version 2.12.0. " + "Please use the functions " + "esmvalcore.preprocessor.cmor_check_metadata, " + "esmvalcore.preprocessor.cmor_check_data, or " + "esmvalcore.cmor.check.cmor_check instead. This function will no " + "longer perform CMOR checks. Fixes and CMOR checks have been " + "clearly separated in ESMValCore version 2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update({ @@ -152,8 +179,12 @@ def fix_metadata(cubes, mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session) - fixed_cubes = [] + session=session, + frequency=frequency) + fixed_cubes = CubeList() + + # Group cubes by input file and apply all fixes to each group element + # (i.e., each file) individually by_file = defaultdict(list) for cube in cubes: by_file[cube.attributes.get('source_file', '')].append(cube) @@ -163,94 +194,97 @@ def fix_metadata(cubes, for fix in fixes: cube_list = fix.fix_metadata(cube_list) - cube = _get_single_cube(cube_list, short_name, project, dataset) - checker = _get_cmor_checker(frequency=frequency, - table=project, - mip=mip, - short_name=short_name, - check_level=check_level, - fail_on_error=False, - automatic_fixes=True) + # The final fix is always GenericFix, whose fix_metadata method always + # returns a single cube + cube = cube_list[0] + + # Perform CMOR checks + # TODO: remove in v2.12 + checker = _get_cmor_checker( + project, + mip, + short_name, + frequency, + fail_on_error=False, + check_level=check_level, + ) cube = checker(cube).check_metadata() + cube.attributes.pop('source_file', None) fixed_cubes.append(cube) - return fixed_cubes - -def _get_single_cube(cube_list, short_name, project, dataset): - if len(cube_list) == 1: - return cube_list[0] - cube = None - for raw_cube in cube_list: - if raw_cube.var_name == short_name: - cube = raw_cube - break - if not cube: - raise ValueError( - f'More than one cube found for variable {short_name} in ' - f'{project}:{dataset} but none of their var_names match the ' - f'expected.\nFull list of cubes encountered: {cube_list}' - ) - logger.warning( - 'Found variable %s in %s:%s, but there were other present in ' - 'the file. Those extra variables are usually metadata ' - '(cell area, latitude descriptions) that was not saved ' - 'according to CF-conventions. It is possible that errors appear ' - 'further on because of this. \nFull list of cubes encountered: %s', - short_name, project, dataset, cube_list) - return cube + return fixed_cubes -def fix_data(cube, - short_name, - project, - dataset, - mip, - frequency=None, - check_level=CheckLevels.DEFAULT, - session: Optional[Session] = None, - **extra_facets): - """Fix cube data if fixes add present and check it anyway. +def fix_data( + cube: Cube, + short_name: str, + project: str, + dataset: str, + mip: str, + frequency: Optional[str] = None, + check_level: CheckLevels = CheckLevels.DEFAULT, + session: Optional[Session] = None, + **extra_facets, +) -> Cube: + """Fix cube data if fixes are required. This method assumes that metadata is already fixed and checked. - This method collects all the relevant fixes for a given variable, applies - them and checks resulting cube (or the original if no fixes were - needed) metadata to ensure that it complies with the standards of its - project CMOR tables. + This method collects all the relevant fixes (including generic ones) for a + given variable and applies them. Parameters ---------- - cube: iris.cube.Cube + cube: Cube to fix. - short_name: str + short_name: Variable's short name. - project: str + project: Project of the dataset. - dataset: str + dataset: Name of the dataset. - mip: str + mip: Variable's MIP. - frequency: str, optional + frequency: Variable's data frequency, if available. - check_level: CheckLevels - Level of strictness of the checks. Set to default. - session: Session, optional + check_level: + Level of strictness of the checks. + + .. deprecated:: 2.10.0 + This option has been deprecated in ESMValCore version 2.10.0 and is + scheduled for removal in version 2.12.0. Please use the functions + :func:`~esmvalcore.preprocessor.cmor_check_metadata`, + :func:`~esmvalcore.preprocessor.cmor_check_data`, or + :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function + will no longer perform CMOR checks. Fixes and CMOR checks have been + clearly separated in ESMValCore version 2.10.0. + session: Current session which includes configuration and directory information. - **extra_facets: dict, optional + **extra_facets: Extra facets are mainly used for data outside of the big projects like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`. Returns ------- - iris.cube.Cube: - Fixed and checked cube. + iris.cube.Cube + Fixed cube. - Raises - ------ - CMORCheckError - If the checker detects errors in the data that it can not fix. """ + # Deprecate CMOR checks (remove in v2.12) + if check_level != CheckLevels.DEFAULT: + msg = ( + "The option `check_level` has been deprecated in ESMValCore " + "version 2.10.0 and is scheduled for removal in version 2.12.0. " + "Please use the functions " + "esmvalcore.preprocessor.cmor_check_metadata, " + "esmvalcore.preprocessor.cmor_check_data, or " + "esmvalcore.cmor.check.cmor_check instead. This function will no " + "longer perform CMOR checks. Fixes and CMOR checks have been " + "clearly separated in ESMValCore version 2.10.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update({ @@ -266,14 +300,20 @@ def fix_data(cube, mip=mip, short_name=short_name, extra_facets=extra_facets, - session=session): + session=session, + frequency=frequency): cube = fix.fix_data(cube) - checker = _get_cmor_checker(frequency=frequency, - table=project, - mip=mip, - short_name=short_name, - fail_on_error=False, - automatic_fixes=True, - check_level=check_level) + + # Perform CMOR checks + # TODO: remove in v2.12 + checker = _get_cmor_checker( + project, + mip, + short_name, + frequency, + fail_on_error=False, + check_level=check_level, + ) cube = checker(cube).check_data() + return cube diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 504a31c137..01659aa206 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -70,18 +70,50 @@ def _get_mips(project: str, short_name: str) -> list[str]: return mips -def get_var_info(project, mip, short_name): +def get_var_info( + project: str, + mip: str, + short_name: str, +) -> VariableInfo | None: """Get variable information. + Note + ---- + If `project=CORDEX` and the `mip` ends with 'hr', it is cropped to 'h' + since CORDEX X-hourly tables define the `mip` as ending in 'h' instead of + 'hr'. + Parameters ---------- - project : str + project: Dataset's project. - mip : str - Variable's cmor table. - short_name : str + mip: + Variable's CMOR table, i.e., MIP. + short_name: Variable's short name. + + Returns + ------- + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + + Raises + ------ + KeyError + No CMOR tables available for `project`. + """ + if project not in CMOR_TABLES: + raise KeyError( + f"No CMOR tables available for project '{project}'. The following " + f"tables are available: {', '.join(CMOR_TABLES)}." + ) + + # CORDEX X-hourly tables define the mip as ending in 'h' instead of 'hr' + if project == 'CORDEX' and mip.endswith('hr'): + mip = mip.replace('hr', 'h') + return CMOR_TABLES[project].get_variable(mip, short_name) @@ -193,6 +225,7 @@ class InfoBase(): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, default, alt_names, strict): if alt_names is None: alt_names = "" @@ -217,28 +250,35 @@ def get_table(self, table): """ return self.tables.get(table) - def get_variable(self, table_name, short_name, derived=False): - """Search and return the variable info. + def get_variable( + self, + table_name: str, + short_name: str, + derived: Optional[bool] = False, + ) -> VariableInfo | None: + """Search and return the variable information. Parameters ---------- - table_name: str - Table name - short_name: str - Variable's short name - derived: bool, optional - Variable is derived. Info retrieval for derived variables always - look on the default tables if variable is not find in the - requested table + table_name: + Table name, i.e., the variable's MIP. + short_name: + Variable's short name. + derived: + Variable is derived. Information retrieval for derived variables + always looks in the default tables (usually, the custom tables) if + variable is not found in the requested table. Returns ------- - VariableInfo - Return the VariableInfo object for the requested variable if - found, returns None if not + VariableInfo | None + `VariableInfo` object for the requested variable if found, ``None`` + otherwise. + """ alt_names_list = self._get_alt_names_list(short_name) + # First, look in requested table table = self.get_table(table_name) if table: for alt_names in alt_names_list: @@ -247,10 +287,19 @@ def get_variable(self, table_name, short_name, derived=False): except KeyError: pass + # If that didn't work, look in all tables (i.e., other MIPs) if + # cmor_strict=False var_info = self._look_in_all_tables(alt_names_list) + + # If that didn' work either, look in default table if cmor_strict=False + # or derived=True if not var_info: var_info = self._look_in_default(derived, alt_names_list, table_name) + + # If necessary, adapt frequency of variable (set it to the one from the + # requested MIP). E.g., if the user asked for table `Amon`, but the + # variable has been found in `day`, use frequency `mon`. if var_info: var_info = var_info.copy() var_info = self._update_frequency_from_mip(table_name, var_info) @@ -258,6 +307,7 @@ def get_variable(self, table_name, short_name, derived=False): return var_info def _look_in_default(self, derived, alt_names_list, table_name): + """Look for variable in default table.""" var_info = None if (not self.strict or derived): for alt_names in alt_names_list: @@ -267,6 +317,7 @@ def _look_in_default(self, derived, alt_names_list, table_name): return var_info def _look_in_all_tables(self, alt_names_list): + """Look for variable in all tables.""" var_info = None if not self.strict: for alt_names in alt_names_list: @@ -276,6 +327,7 @@ def _look_in_all_tables(self, alt_names_list): return var_info def _get_alt_names_list(self, short_name): + """Get list of alternative variable names.""" alt_names_list = [short_name] for alt_names in self.alt_names: if short_name in alt_names: @@ -286,12 +338,14 @@ def _get_alt_names_list(self, short_name): return alt_names_list def _update_frequency_from_mip(self, table_name, var_info): + """Update frequency information of var_info from table.""" mip_info = self.get_table(table_name) if mip_info: var_info.frequency = mip_info.frequency return var_info def _look_all_tables(self, alt_names): + """Look for variable in all tables.""" for table_vars in sorted(self.tables.values()): if alt_names in table_vars: return table_vars[alt_names] @@ -315,6 +369,7 @@ class CMIP6Info(InfoBase): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, cmor_tables_path, default=None, @@ -471,6 +526,7 @@ def _is_table(table_data): @total_ordering class TableInfo(dict): """Container class for storing a CMOR table.""" + def __init__(self, *args, **kwargs): """Create a new TableInfo object for storing VariableInfo objects.""" super(TableInfo, self).__init__(*args, **kwargs) @@ -496,6 +552,7 @@ class JsonInfo(object): Provides common utility methods to read json variables """ + def __init__(self): self._json_data = {} @@ -536,6 +593,7 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" + def __init__(self, table_type, short_name): """Class to read and store variable information. @@ -647,6 +705,7 @@ def has_coord_with_standard_name(self, standard_name: str) -> bool: class CoordinateInfo(JsonInfo): """Class to read and store coordinate information.""" + def __init__(self, name): """Class to read and store coordinate information. @@ -734,6 +793,7 @@ class CMIP5Info(InfoBase): If False, will look for a variable in other tables if it can not be found in the requested one """ + def __init__(self, cmor_tables_path, default=None, @@ -892,6 +952,7 @@ class CMIP3Info(CMIP5Info): If False, will look for a variable in other tables if it can not be found in the requested one """ + def _read_table_file(self, table_file, table=None): for dim in ('zlevel', ): coord = CoordinateInfo(dim) diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 265004a8cf..e49afd4785 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -22,7 +22,7 @@ from iris.cube import Cube, CubeList from iris.time import PartialDateTime -from esmvalcore.cmor.check import _get_next_month, _get_time_bounds +from esmvalcore.cmor._fixes.fix import get_next_month, get_time_bounds from esmvalcore.iris_helpers import date2num from ._shared import get_iris_analysis_operation, operator_accept_weights @@ -128,7 +128,7 @@ def _parse_end_date(date): if len(date) == 4: end_date = datetime.datetime(int(date) + 1, 1, 1, 0, 0, 0) elif len(date) == 6: - month, year = _get_next_month(int(date[4:]), int(date[0:4])) + month, year = get_next_month(int(date[4:]), int(date[0:4])) end_date = datetime.datetime(year, month, 1, 0, 0, 0) else: try: @@ -966,7 +966,7 @@ def regrid_time(cube: Cube, frequency: str) -> Cube: # uniformize bounds cube.coord('time').bounds = None - cube.coord('time').bounds = _get_time_bounds(cube.coord('time'), frequency) + cube.coord('time').bounds = get_time_bounds(cube.coord('time'), frequency) # remove aux coords that will differ reset_aux = ['day_of_month', 'day_of_year'] diff --git a/tests/integration/cmor/_fixes/cesm/test_cesm2.py b/tests/integration/cmor/_fixes/cesm/test_cesm2.py index 44fed835b7..661eadcfb7 100644 --- a/tests/integration/cmor/_fixes/cesm/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cesm/test_cesm2.py @@ -7,6 +7,7 @@ from iris.cube import Cube, CubeList import esmvalcore.cmor._fixes.cesm.cesm2 +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import CoordinateInfo, get_var_info from esmvalcore.config._config import get_extra_facets @@ -338,6 +339,7 @@ def test_get_tas_fix(): fix = Fix.get_fixes('CESM', 'CESM2', 'Amon', 'tas') assert fix == [ esmvalcore.cmor._fixes.cesm.cesm2.AllVars(None), + GenericFix(None), ] diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py index 1312762aab..3cba189b7a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py @@ -8,7 +8,7 @@ from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.cmip5.access1_0 import AllVars, Cl -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor.table import get_var_info from esmvalcore.iris_helpers import date2num @@ -39,8 +39,9 @@ class TestAllVars: @staticmethod def test_get(): """Test getting of fix.""" - assert (Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas') - == [AllVars(None)]) + assert Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas') == [ + AllVars(None), GenericFix(None) + ] @staticmethod def test_fix_metadata(cube): @@ -66,7 +67,7 @@ def test_fix_metadata_if_not_time(cube): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py index 68d59201de..684e88f65f 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py @@ -8,7 +8,7 @@ from esmvalcore.cmor._fixes.cmip5.access1_0 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip5.access1_3 import AllVars, Cl -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.iris_helpers import date2num @@ -38,8 +38,9 @@ class TestAllVars: @staticmethod def test_get(): """Test getting of fix.""" - assert (Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas') - == [AllVars(None)]) + assert Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas') == [ + AllVars(None), GenericFix(None) + ] @staticmethod def test_fix_metadata(cube): @@ -65,7 +66,7 @@ def test_fix_metadata_if_not_time(cube): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py index 48d846146d..65f37766e0 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py @@ -1,5 +1,4 @@ """Test bcc-csm1-1 fixes.""" -import unittest import iris import numpy as np @@ -9,6 +8,7 @@ ClFixHybridPressureCoord, OceanFixGrid, ) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -16,7 +16,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'bcc-csm1-1', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -24,13 +24,10 @@ def test_cl_fix(): assert Cl is ClFixHybridPressureCoord -class TestTos(unittest.TestCase): - """Test tos fixes.""" - - def test_get(self): - """Test fix get.""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'bcc-csm1-1', 'Amon', 'tos'), [Tos(None)]) +def test_get_tos_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP5', 'bcc-csm1-1', 'Omon', 'tos') + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py index 8473c84f44..8fe017fe9b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py @@ -1,18 +1,17 @@ """Test fixes for bcc-csm1-1-m.""" -import unittest from esmvalcore.cmor._fixes.cmip5.bcc_csm1_1_m import Cl, Tos from esmvalcore.cmor._fixes.common import ( ClFixHybridPressureCoord, OceanFixGrid, ) -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'bcc-csm1-1-m', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -20,13 +19,10 @@ def test_cl_fix(): assert Cl is ClFixHybridPressureCoord -class TestTos(unittest.TestCase): - """Test tos fixes.""" - - def test_get(self): - """Test fix get.""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'bcc-csm1-1-m', 'Amon', 'tos'), [Tos(None)]) +def test_get_tos_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP5', 'bcc-csm1-1-m', 'Omon', 'tos') + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py index e8045902f5..d988e02441 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py @@ -5,9 +5,16 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor._fixes.cmip5.bnu_esm import (Ch4, Cl, Co2, FgCo2, - Od550Aer, SpCo2) +from esmvalcore.cmor._fixes.cmip5.bnu_esm import ( + Ch4, + Cl, + Co2, + FgCo2, + Od550Aer, + SpCo2, +) from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -22,7 +29,7 @@ def setUp(self): def test_get(self): """Test fix get""" fix = Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(self): """Test fix for ``cl``.""" @@ -46,7 +53,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'co2'), - [Co2(self.vardef)]) + [Co2(self.vardef), GenericFix(None)]) def test_fix_metadata(self): """Test unit change.""" @@ -74,7 +81,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'fgco2'), - [FgCo2(self.vardef)]) + [FgCo2(self.vardef), GenericFix(None)]) def test_fix_metadata(self): """Test unit fix.""" @@ -101,7 +108,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), - [Ch4(self.vardef)]) + [Ch4(self.vardef), GenericFix(None)]) def test_fix_metadata(self): """Test unit fix.""" @@ -127,7 +134,8 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'spco2'), [SpCo2(None)]) + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'spco2'), + [SpCo2(None), GenericFix(None)]) def test_fix_metadata(self): """Test fix.""" @@ -157,7 +165,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'od550aer'), - [Od550Aer(None)]) + [Od550Aer(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py index 27d29839ab..4e8f9c826d 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py @@ -6,13 +6,14 @@ from esmvalcore.cmor._fixes.cmip5.canesm2 import Cl, FgCo2 from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -31,7 +32,8 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CANESM2', 'Amon', 'fgco2'), [FgCo2(None)]) + Fix.get_fixes('CMIP5', 'CANESM2', 'Amon', 'fgco2'), + [FgCo2(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py index f19cc98165..05bb6620a0 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py @@ -7,6 +7,7 @@ from esmvalcore.cmor._fixes.cmip5.bnu_esm import Cl as BaseCl from esmvalcore.cmor._fixes.cmip5.ccsm4 import AllVars, Cl, Csoil, So +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -59,7 +60,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'rlut'), - [AllVars(None)]) + [AllVars(None), GenericFix(None)]) def test_fix_metadata(self): """Check that latitudes values are rounded.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py index 31fb3f670f..fdb2314d80 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_bgc.py @@ -6,13 +6,14 @@ from esmvalcore.cmor._fixes.cmip5.cesm1_bgc import Cl, Gpp, Nbp from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl as BaseCl +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -31,7 +32,8 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp'), [Gpp(None)]) + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp'), + [Gpp(None), GenericFix(None)]) def test_fix_data(self): """Test fix to set missing values correctly.""" @@ -54,4 +56,5 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'nbp'), [Nbp(None)]) + Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'nbp'), + [Nbp(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_cam5.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_cam5.py index d5991ddff6..07cf8ab1a9 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_cam5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_cam5.py @@ -3,13 +3,14 @@ import pytest from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CESM1-CAM5', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_fastchem.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_fastchem.py index a41d1996f1..6ae2a44913 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_fastchem.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_fastchem.py @@ -1,13 +1,14 @@ """Tests for CESM1-FASTCHEM fixes.""" from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip5.cesm1_fastchem import Cl +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CESM1-FASTCHEM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_cesm1_waccm.py b/tests/integration/cmor/_fixes/cmip5/test_cesm1_waccm.py index 20023eebfa..146a456dad 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cesm1_waccm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cesm1_waccm.py @@ -1,13 +1,14 @@ """Tests for CESM1-WACCM fixes.""" from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl as BaseCl from esmvalcore.cmor._fixes.cmip5.cesm1_waccm import Cl +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CESM1-WACCM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py index ae9da2ef04..ce6ebe0121 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py @@ -4,8 +4,9 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.cnrm_cm5 import Msftmyz, Msftmyzba +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor.fix import Fix class TestMsftmyz(unittest.TestCase): @@ -19,7 +20,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyz'), - [Msftmyz(None)]) + [Msftmyz(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -39,7 +40,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'CNRM-CM5', 'Amon', 'msftmyzba'), - [Msftmyzba(None)]) + [Msftmyzba(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_csiro_mk3_6_0.py b/tests/integration/cmor/_fixes/cmip5/test_csiro_mk3_6_0.py index a7d23d9d41..3fa4318d72 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_csiro_mk3_6_0.py +++ b/tests/integration/cmor/_fixes/cmip5/test_csiro_mk3_6_0.py @@ -1,13 +1,13 @@ """Test fixes for CSIRO-Mk3-6-0.""" from esmvalcore.cmor._fixes.cmip5.csiro_mk3_6_0 import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'CSIRO-Mk3-6-0', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py index 8ec734a90c..4b480ef81f 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py @@ -2,18 +2,19 @@ import unittest import numpy as np - from cf_units import Unit from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList from iris.exceptions import CoordinateNotFoundError + from esmvalcore.cmor._fixes.cmip5.ec_earth import ( Areacello, Pr, Sftlf, Sic, Tas, - ) +) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -27,7 +28,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sic'), - [Sic(None)]) + [Sic(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -46,7 +47,8 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sftlf'), [Sftlf(None)]) + Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'sftlf'), + [Sftlf(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -90,7 +92,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'tas'), - [Tas(None)]) + [Tas(None), GenericFix(None)]) def test_tas_fix_metadata(self): """Test metadata fix.""" @@ -148,7 +150,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'EC-EARTH', 'Omon', 'areacello'), - [Areacello(None)], + [Areacello(None), GenericFix(None)], ) def test_areacello_fix_metadata(self): @@ -210,7 +212,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'EC-EARTH', 'Amon', 'pr'), - [Pr(None)], + [Pr(None), GenericFix(None)], ) def test_pr_fix_metadata(self): diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py index abda3d180b..b70b16fa25 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py @@ -5,6 +5,7 @@ from iris.cube import Cube from esmvalcore.cmor._fixes.cmip5.fgoals_g2 import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -33,8 +34,9 @@ class TestAll: @staticmethod def test_get(): """Test fix get.""" - assert (Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas') - == [AllVars(None)]) + assert Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas') == [ + AllVars(None), GenericFix(None) + ] @staticmethod def test_fix_metadata(cube): diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_s2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_s2.py index e77c095b3c..a1c15afebd 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_s2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_s2.py @@ -4,13 +4,14 @@ from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.cmip5.fgoals_s2 import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_allvars_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'FGOALS-s2', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] LAT_COORD = DimCoord( diff --git a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py index b8b8429f33..c0705e58fa 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py @@ -4,15 +4,16 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.fio_esm import Ch4, Cl, Co2 from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl as BaseCl +from esmvalcore.cmor._fixes.cmip5.fio_esm import Ch4, Cl, Co2 +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -30,7 +31,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'ch4'), - [Ch4(None)]) + [Ch4(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -49,7 +50,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'FIO-ESM', 'Amon', 'co2'), - [Co2(None)]) + [Co2(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py index 63d5ce426d..8a4fa19fd1 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py @@ -6,9 +6,15 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor._fixes.cmip5.gfdl_cm2p1 import (AllVars, Areacello, Cl, - Sftof, Sit) from esmvalcore.cmor._fixes.cmip5.cesm1_cam5 import Cl as BaseCl +from esmvalcore.cmor._fixes.cmip5.gfdl_cm2p1 import ( + AllVars, + Areacello, + Cl, + Sftof, + Sit, +) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -19,7 +25,7 @@ def test_get(self): """Test getting of fix.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'cl'), - [Cl(None), AllVars(None)]) + [Cl(None), AllVars(None), GenericFix(None)]) def test_fix(self): """Test fix for ``cl``.""" @@ -37,7 +43,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'fx', 'sftof'), - [Sftof(None), AllVars(None)]) + [Sftof(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -59,7 +65,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'areacello'), - [Areacello(self.vardef), AllVars(self.vardef)]) + [Areacello(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" @@ -101,7 +109,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'OImon', 'sit'), - [Sit(self.var_info_mock), AllVars(None)]) + [Sit(self.var_info_mock), AllVars(None), GenericFix(None)]) def test_fix_metadata_day_do_nothing(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py index c4aaae2729..bcda26c95b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py @@ -5,6 +5,7 @@ from iris.cube import Cube from esmvalcore.cmor._fixes.cmip5.gfdl_cm3 import AllVars, Areacello, Sftof +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -20,7 +21,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM3', 'fx', 'sftof'), - [Sftof(None), AllVars(None)]) + [Sftof(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -41,7 +42,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'areacello'), - [Areacello(self.vardef), AllVars(self.vardef)]) + [Areacello(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py index 89a1a1e32b..d5189e84c4 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py @@ -7,9 +7,16 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor._fixes.cmip5.gfdl_esm2g import (AllVars, Areacello, Co2, - FgCo2, Usi, Vsi, - _get_and_remove) +from esmvalcore.cmor._fixes.cmip5.gfdl_esm2g import ( + AllVars, + Areacello, + Co2, + FgCo2, + Usi, + Vsi, + _get_and_remove, +) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -71,7 +78,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'co2'), - [Co2(None), AllVars(None)]) + [Co2(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -92,7 +99,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'day', 'usi'), - [Usi(self.vardef), AllVars(self.vardef)]) + [Usi(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_data(self): """Test metadata fix.""" @@ -112,7 +121,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'day', 'vsi'), - [Vsi(self.vardef), AllVars(self.vardef)]) + [Vsi(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_data(self): """Test metadata fix.""" @@ -132,7 +143,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'fx', 'areacello'), - [Areacello(self.vardef), AllVars(self.vardef)]) + [Areacello(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py index 1538098d25..1897ba5c85 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py @@ -4,8 +4,13 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor._fixes.cmip5.gfdl_esm2m import (AllVars, Areacello, Co2, - Sftof) +from esmvalcore.cmor._fixes.cmip5.gfdl_esm2m import ( + AllVars, + Areacello, + Co2, + Sftof, +) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -21,7 +26,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'fx', 'sftof'), - [Sftof(None), AllVars(None)]) + [Sftof(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -41,7 +46,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'co2'), - [Co2(None), AllVars(None)]) + [Co2(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -62,7 +67,9 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'fx', 'areacello'), - [Areacello(self.vardef), AllVars(self.vardef)]) + [Areacello(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_giss_e2_h.py b/tests/integration/cmor/_fixes/cmip5/test_giss_e2_h.py index 5fd78f648c..9c39dd136f 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_giss_e2_h.py +++ b/tests/integration/cmor/_fixes/cmip5/test_giss_e2_h.py @@ -1,13 +1,13 @@ """Test fixes for GISS-E2-H.""" from esmvalcore.cmor._fixes.cmip5.giss_e2_h import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'GISS-E2-H', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_giss_e2_r.py b/tests/integration/cmor/_fixes/cmip5/test_giss_e2_r.py index e030bb4aaf..d95b517cdd 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_giss_e2_r.py +++ b/tests/integration/cmor/_fixes/cmip5/test_giss_e2_r.py @@ -1,13 +1,13 @@ """Test fixes for GISS-E2-R.""" from esmvalcore.cmor._fixes.cmip5.giss_e2_r import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'GISS-E2-R', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py index ed2f7c0d1f..0e6a4fc57a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py @@ -1,8 +1,9 @@ """Test HADGEM2-CC fixes.""" import unittest +from esmvalcore.cmor._fixes.cmip5.hadgem2_cc import O2, AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.hadgem2_cc import AllVars, O2 class TestAllVars(unittest.TestCase): @@ -11,7 +12,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'tas'), - [AllVars(None)]) + [AllVars(None), GenericFix(None)]) class TestO2(unittest.TestCase): @@ -20,4 +21,4 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'HADGEM2-CC', 'Amon', 'o2'), - [O2(None), AllVars(None)]) + [O2(None), AllVars(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py index 99c7eb4aa9..c0ab2bb8fd 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py @@ -3,6 +3,7 @@ from esmvalcore.cmor._fixes.cmip5.hadgem2_es import O2, AllVars, Cl from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -13,7 +14,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'tas'), - [AllVars(None)]) + [AllVars(None), GenericFix(None)]) class TestO2(unittest.TestCase): @@ -23,13 +24,13 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'HADGEM2-ES', 'Amon', 'o2'), - [O2(None), AllVars(None)]) + [O2(None), AllVars(None), GenericFix(None)]) def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'HadGEM2-ES', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py index a5a5cf0235..dcabbbf313 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py @@ -6,13 +6,14 @@ from esmvalcore.cmor._fixes.cmip5.inmcm4 import Cl, Gpp, Lai, Nbp from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'inmcm4', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -31,7 +32,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'gpp'), - [Gpp(None)]) + [Gpp(None), GenericFix(None)]) def test_fix_data(self): """Test data fox.""" @@ -51,7 +52,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'lai'), - [Lai(None)]) + [Lai(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -71,7 +72,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'INMCM4', 'Amon', 'nbp'), - [Nbp(None)]) + [Nbp(None), GenericFix(None)]) def test_fix_metadata(self): """Test fix on nbp files to set standard_name.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_lr.py b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_lr.py index 443a4aafbb..67f50c1ea0 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_lr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_lr.py @@ -1,13 +1,13 @@ """Test fixes for IPSL-CM5A-LR.""" from esmvalcore.cmor._fixes.cmip5.ipsl_cm5a_lr import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'IPSL-CM5A-LR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_mr.py b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_mr.py index ff3e0adc25..787a2d804a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_mr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5a_mr.py @@ -1,13 +1,13 @@ """Test fixes for IPSL-CM5A-MR.""" from esmvalcore.cmor._fixes.cmip5.ipsl_cm5a_mr import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'IPSL-CM5A-MR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5b_lr.py b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5b_lr.py index e04fa4718a..4a7b0a4476 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5b_lr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ipsl_cm5b_lr.py @@ -1,13 +1,13 @@ """Test fixes for IPSL-CM5B-LR.""" from esmvalcore.cmor._fixes.cmip5.ipsl_cm5b_lr import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'IPSL-CM5B-LR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py index a9d61dde10..63a706eef7 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc5.py @@ -7,13 +7,14 @@ from esmvalcore.cmor._fixes.cmip5.miroc5 import Cl, Hur, Pr, Sftof, Tas from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -24,13 +25,13 @@ def test_cl_fix(): def test_get_hur_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'hur') - assert fix == [Hur(None)] + assert fix == [Hur(None), GenericFix(None)] def test_get_pr_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'pr') - assert fix == [Pr(None)] + assert fix == [Pr(None), GenericFix(None)] @unittest.mock.patch( @@ -64,7 +65,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'sftof'), - [Sftof(None)]) + [Sftof(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -88,7 +89,7 @@ def setUp(self): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('CMIP5', 'MIROC5', 'Amon', 'tas'), - [Tas(None)]) + [Tas(None), GenericFix(None)]) def test_fix_metadata(self): """Test metadata fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py index f87902a8d9..1010e4e670 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py @@ -9,6 +9,7 @@ from esmvalcore.cmor._fixes.cmip5.miroc_esm import AllVars, Cl, Co2, Tro3 from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -16,7 +17,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): @@ -37,7 +38,9 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'co2'), - [Co2(self.vardef), AllVars(self.vardef)]) + [Co2(self.vardef), + AllVars(self.vardef), + GenericFix(self.vardef)]) def test_fix_metadata(self): """Test unit fix.""" @@ -58,7 +61,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tro3'), - [Tro3(None), AllVars(None)]) + [Tro3(None), AllVars(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -102,7 +105,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'tos'), - [AllVars(None)]) + [AllVars(None), GenericFix(None)]) def test_fix_metadata_plev(self): """Test plev fix.""" @@ -112,7 +115,7 @@ def test_fix_metadata_plev(self): cube.coord('air_pressure') def test_fix_metadata_no_plev(self): - """Test plev fix wotk with no plev.""" + """Test plev fix work with no plev.""" self.cube.remove_coord('AR5PL35') cube = self.fix.fix_metadata([self.cube])[0] with self.assertRaises(CoordinateNotFoundError): diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py index 186f321710..1e14dfd30c 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py @@ -5,8 +5,9 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.miroc_esm_chem import Tro3 +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor.fix import Fix class TestTro3(unittest.TestCase): @@ -20,7 +21,7 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MIROC-ESM-CHEM', 'Amon', 'tro3'), - [Tro3(None)]) + [Tro3(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py index effb781f9f..07aa974d15 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py @@ -6,13 +6,14 @@ from esmvalcore.cmor._fixes.cmip5.mpi_esm_lr import Cl, Pctisccp from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -32,7 +33,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MPI-ESM-LR', 'Amon', 'pctisccp'), - [Pctisccp(None)]) + [Pctisccp(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_mr.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_mr.py index 1c074d41a6..96cd83b963 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_mr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_mr.py @@ -1,13 +1,13 @@ """Test fixes for MPI-ESM-MR.""" from esmvalcore.cmor._fixes.cmip5.mpi_esm_mr import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MPI-ESM-MR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_p.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_p.py index 7e0aa87a5a..2466e5a9f8 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_p.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_p.py @@ -1,13 +1,13 @@ """Test fixes for MPI-ESM-P.""" from esmvalcore.cmor._fixes.cmip5.mpi_esm_p import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MPI-ESM-P', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py index 2b0d85ab7a..717c66cee6 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py @@ -3,13 +3,14 @@ from esmvalcore.cmor._fixes.cmip5.mri_cgcm3 import Cl, Msftmyz, ThetaO from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -24,7 +25,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'msftmyz'), - [Msftmyz(None)]) + [Msftmyz(None), GenericFix(None)]) class TestThetao(unittest.TestCase): @@ -34,4 +35,4 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MRI-CGCM3', 'Amon', 'thetao'), - [ThetaO(None)]) + [ThetaO(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py index c00af1d6fe..f4d39eb70b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py @@ -1,8 +1,9 @@ """Test MRI-ESM1 fixes.""" import unittest -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.mri_esm1 import Msftmyz +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor.fix import Fix class TestMsftmyz(unittest.TestCase): @@ -11,4 +12,4 @@ def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MRI-ESM1', 'Amon', 'msftmyz'), - [Msftmyz(None)]) + [Msftmyz(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_m.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_m.py index 79664ce6b9..1ad165c633 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_m.py @@ -1,13 +1,13 @@ """Test fixes for NorESM1-M.""" from esmvalcore.cmor._fixes.cmip5.noresm1_m import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP5', 'NorESM1-M', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py index ab286d78bd..b600311d5a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py +++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py @@ -1,10 +1,11 @@ """Tests for fixes of NorESM1-ME (CMIP5).""" -import pytest import iris +import pytest from iris.cube import CubeList -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.noresm1_me import Tas +from esmvalcore.cmor._fixes.fix import GenericFix +from esmvalcore.cmor.fix import Fix DIM_COORD_SHORT = iris.coords.DimCoord( [1.0, 2.0, 3.0], @@ -69,4 +70,6 @@ def test_tas(cubes_in, cubes_out): def test_get(): """Test fix get""" - assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'Amon', 'tas') == [Tas(None)] + assert Fix.get_fixes('CMIP5', 'NORESM1-ME', 'Amon', 'tas') == [ + Tas(None), GenericFix(None) + ] diff --git a/tests/integration/cmor/_fixes/cmip6/test_access_cm2.py b/tests/integration/cmor/_fixes/cmip6/test_access_cm2.py index 0e5bd66eb9..00977403ea 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_access_cm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_access_cm2.py @@ -7,6 +7,7 @@ from esmvalcore.cmor._fixes.cmip6.access_cm2 import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix B_POINTS = [ @@ -108,7 +109,7 @@ def cl_cubes(): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-CM2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -132,7 +133,7 @@ def test_cl_fix_metadata(mock_base_fix_metadata, cl_cubes): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-CM2', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -143,7 +144,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-CM2', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_access_esm1_5.py b/tests/integration/cmor/_fixes/cmip6/test_access_esm1_5.py index d126d3a74d..29fb72c870 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_access_esm1_5.py +++ b/tests/integration/cmor/_fixes/cmip6/test_access_esm1_5.py @@ -7,6 +7,7 @@ from esmvalcore.cmor._fixes.cmip6.access_esm1_5 import Cl, Cli, Clw, Hus, Zg from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -77,7 +78,7 @@ def cl_cubes(): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-ESM1-5', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] @unittest.mock.patch( @@ -101,7 +102,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-ESM1-5', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -112,7 +113,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-ESM1-5', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -143,7 +144,7 @@ def cubes_with_wrong_air_pressure(): def test_get_hus_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-ESM1-5', 'Amon', 'hus') - assert fix == [Hus(None)] + assert fix == [Hus(None), GenericFix(None)] def test_hus_fix_metadata(cubes_with_wrong_air_pressure): @@ -169,7 +170,7 @@ def test_hus_fix_metadata(cubes_with_wrong_air_pressure): def test_get_zg_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ACCESS-ESM1-5', 'Amon', 'zg') - assert fix == [Zg(None)] + assert fix == [Zg(None), GenericFix(None)] def test_zg_fix_metadata(cubes_with_wrong_air_pressure): diff --git a/tests/integration/cmor/_fixes/cmip6/test_awi_cm_1_1_mr.py b/tests/integration/cmor/_fixes/cmip6/test_awi_cm_1_1_mr.py index f198e2f69b..780b09bd7f 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_awi_cm_1_1_mr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_awi_cm_1_1_mr.py @@ -3,6 +3,7 @@ import pytest from esmvalcore.cmor._fixes.cmip6.awi_cm_1_1_mr import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -34,7 +35,7 @@ def cubes(): def test_get_allvars_fix(): fix = Fix.get_fixes('CMIP6', 'AWI-CM-1-1-MR', 'Amon', 'wrong_lat_lname') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_awi_esm_1_1_lr.py b/tests/integration/cmor/_fixes/cmip6/test_awi_esm_1_1_lr.py index eba75631e2..07dd1f41ef 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_awi_esm_1_1_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_awi_esm_1_1_lr.py @@ -3,6 +3,7 @@ import pytest from esmvalcore.cmor._fixes.cmip6.awi_esm_1_1_lr import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -15,7 +16,7 @@ def sample_cubes(): def test_get_tas_fix(): fix = Fix.get_fixes('CMIP6', 'AWI-ESM-1-1-LR', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(sample_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py b/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py index ee5739380f..5281953fa0 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py @@ -12,13 +12,13 @@ ClFixHybridPressureCoord, OceanFixGrid, ) -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_areacello_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Amon', 'areacello') - assert fix == [Areacello(None)] + assert fix == [Areacello(None), GenericFix(None)] def test_areacello_fix(): @@ -29,7 +29,7 @@ def test_areacello_fix(): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -40,7 +40,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -51,7 +51,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -62,7 +62,7 @@ def test_clw_fix(): def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Omon', 'tos') - assert fix == [Tos(None)] + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix(): @@ -73,7 +73,7 @@ def test_tos_fix(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -84,7 +84,7 @@ def test_siconc_fix(): def test_get_sos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Omon', 'sos') - assert fix == [Sos(None)] + assert fix == [Sos(None), GenericFix(None)] def test_sos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_bcc_esm1.py b/tests/integration/cmor/_fixes/cmip6/test_bcc_esm1.py index 382d0268f3..076390e923 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_bcc_esm1.py +++ b/tests/integration/cmor/_fixes/cmip6/test_bcc_esm1.py @@ -11,13 +11,13 @@ ClFixHybridPressureCoord, OceanFixGrid, ) -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -28,7 +28,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -39,7 +39,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -50,7 +50,7 @@ def test_clw_fix(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -61,7 +61,7 @@ def test_siconc_fix(): def test_get_sos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'Omon', 'sos') - assert fix == [Sos(None)] + assert fix == [Sos(None), GenericFix(None)] def test_sos_fix(): @@ -72,7 +72,7 @@ def test_sos_fix(): def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-ESM1', 'Omon', 'tos') - assert fix == [Tos(None)] + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cams_csm1_0.py b/tests/integration/cmor/_fixes/cmip6/test_cams_csm1_0.py index 55cb512aee..eb2367539c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cams_csm1_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cams_csm1_0.py @@ -1,13 +1,13 @@ """Test fixes for CAMS-CSM1-0.""" from esmvalcore.cmor._fixes.cmip6.cams_csm1_0 import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CAMS-CSM1-0', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +18,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CAMS-CSM1-0', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +29,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CAMS-CSM1-0', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_canesm5.py b/tests/integration/cmor/_fixes/cmip6/test_canesm5.py index 9b9638e5f3..5f23af82f3 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_canesm5.py +++ b/tests/integration/cmor/_fixes/cmip6/test_canesm5.py @@ -4,13 +4,14 @@ import pytest from esmvalcore.cmor._fixes.cmip6.canesm5 import Co2, Gpp +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_co2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CanESM5', 'Amon', 'co2') - assert fix == [Co2(None)] + assert fix == [Co2(None), GenericFix(None)] @pytest.fixture @@ -48,7 +49,7 @@ def gpp_cube(): def test_get_gpp_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CanESM5', 'Lmon', 'gpp') - assert fix == [Gpp(None)] + assert fix == [Gpp(None), GenericFix(None)] def test_gpp_fix_data(gpp_cube): diff --git a/tests/integration/cmor/_fixes/cmip6/test_canesm5_canoe.py b/tests/integration/cmor/_fixes/cmip6/test_canesm5_canoe.py index 349444559c..f0f43fd597 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_canesm5_canoe.py +++ b/tests/integration/cmor/_fixes/cmip6/test_canesm5_canoe.py @@ -2,13 +2,13 @@ from esmvalcore.cmor._fixes.cmip6.canesm5 import Co2 as BaseCo2 from esmvalcore.cmor._fixes.cmip6.canesm5 import Gpp as BaseGpp from esmvalcore.cmor._fixes.cmip6.canesm5_canoe import Co2, Gpp -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_co2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CanESM5-CanOE', 'Amon', 'co2') - assert fix == [Co2(None)] + assert fix == [Co2(None), GenericFix(None)] def test_co2_fix(): @@ -19,7 +19,7 @@ def test_co2_fix(): def test_get_gpp_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CanESM5-CanOE', 'Lmon', 'gpp') - assert fix == [Gpp(None)] + assert fix == [Gpp(None), GenericFix(None)] def test_gpp_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cas_esm2_0.py b/tests/integration/cmor/_fixes/cmip6/test_cas_esm2_0.py index bd018b3f2e..2e22f91c1d 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cas_esm2_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cas_esm2_0.py @@ -1,13 +1,14 @@ """Tests for the fixes of CAS-ESM2-0.""" from esmvalcore.cmor._fixes.cmip6.cas_esm2_0 import Cl from esmvalcore.cmor._fixes.cmip6.ciesm import Cl as BaseCl +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CAS-ESM2-0', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 55442a323a..e60e1727b2 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -19,6 +19,7 @@ Tos, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -26,7 +27,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] AIR_PRESSURE_POINTS = np.array([[[[1.0, 1.0, 1.0, 1.0], @@ -163,7 +164,7 @@ def test_cl_fix_metadata(cl_cubes): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -174,7 +175,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -268,31 +269,31 @@ def thetao_cubes(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'tos') - assert fix == [Tos(None), Omon(None)] + assert fix == [Tos(None), Omon(None), GenericFix(None)] def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'thetao') - assert fix == [Omon(None)] + assert fix == [Omon(None), GenericFix(None)] def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_tas_fix_metadata(tas_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py index 96a2013075..16d8a9c0b2 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_fv2.py @@ -12,13 +12,14 @@ Tas, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -29,7 +30,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cli_fix(): @@ -40,7 +41,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -51,7 +52,7 @@ def test_clw_fix(): def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_fgco2_fix(): @@ -62,7 +63,7 @@ def test_fgco2_fix(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -73,7 +74,7 @@ def test_siconc_fix(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-FV2', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index b9075b6716..b80beb45fc 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -20,13 +20,14 @@ Tas, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -65,7 +66,7 @@ def test_cl_fix_file(mock_get_filepath, tmp_path, test_data_path): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -76,7 +77,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -87,7 +88,7 @@ def test_clw_fix(): def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_fgco2_fix(): @@ -98,7 +99,7 @@ def test_fgco2_fix(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -117,7 +118,7 @@ def tas_cubes(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py index a5ac8f6735..d4eb9b4b62 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm_fv2.py @@ -1,6 +1,6 @@ """Tests for the fixes of CESM2-WACCM-FV2.""" -from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2 import Fgco2 as BaseFgco2 +from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import Cl as BaseCl from esmvalcore.cmor._fixes.cmip6.cesm2_waccm_fv2 import ( Cl, @@ -12,13 +12,14 @@ Tas, ) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -29,7 +30,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -40,7 +41,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -51,7 +52,7 @@ def test_clw_fix(): def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_fgco2_fix(): @@ -62,7 +63,7 @@ def test_fgco2_fix(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -73,7 +74,7 @@ def test_siconc_fix(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2-WACCM-FV2', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_ciesm.py b/tests/integration/cmor/_fixes/cmip6/test_ciesm.py index 29da65c025..1cf1aa3aad 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ciesm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ciesm.py @@ -4,13 +4,14 @@ from esmvalcore.cmor._fixes.cmip6.ciesm import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CIESM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/cmip6/test_cmcc_cm2_sr5.py b/tests/integration/cmor/_fixes/cmip6/test_cmcc_cm2_sr5.py index 44f410427c..27aa306289 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cmcc_cm2_sr5.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cmcc_cm2_sr5.py @@ -6,6 +6,7 @@ from esmvalcore.cmor._fixes.cmip6.cmcc_cm2_sr5 import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -13,7 +14,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CMCC-CM2-SR5', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1.py b/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1.py index 8322b1e7fe..18ca8a3fa4 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1.py @@ -4,8 +4,14 @@ import numpy as np import pytest -from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import (Cl, Clcalipso, - Cli, Clw, Omon) +from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import ( + Cl, + Clcalipso, + Cli, + Clw, + Omon, +) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -14,7 +20,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] AIR_PRESSURE_POINTS = np.array([[[[1.0, 1.0], @@ -85,7 +91,7 @@ def test_cl_fix_metadata(test_data_path): def test_get_clcalipso_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1', 'CFmon', 'clcalipso') - assert fix == [Clcalipso(None)] + assert fix == [Clcalipso(None), GenericFix(None)] @pytest.fixture @@ -113,7 +119,7 @@ def test_clcalipso_fix_metadata(clcalipso_cubes): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -124,7 +130,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -135,4 +141,4 @@ def test_clw_fix(): def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1', 'Omon', 'thetao') - assert fix == [Omon(None)] + assert fix == [Omon(None), GenericFix(None)] diff --git a/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1_hr.py b/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1_hr.py index fa43c06acf..9ac3e8bad1 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1_hr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cnrm_cm6_1_hr.py @@ -3,13 +3,13 @@ from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import Cli as BaseCli from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import Clw as BaseClw from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1_hr import Cl, Cli, Clw -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1-HR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -20,7 +20,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1-HR', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -31,7 +31,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-CM6-1-HR', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_cnrm_esm2_1.py b/tests/integration/cmor/_fixes/cmip6/test_cnrm_esm2_1.py index 8ff7b7e0a2..3e6d66ebb9 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cnrm_esm2_1.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cnrm_esm2_1.py @@ -7,16 +7,21 @@ from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import Clcalipso as BaseClcalipso from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import Cli as BaseCli from esmvalcore.cmor._fixes.cmip6.cnrm_cm6_1 import Clw as BaseClw -from esmvalcore.cmor._fixes.cmip6.cnrm_esm2_1 import (Cl, Clcalipso, - Cli, Clw, Omon) -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.cmip6.cnrm_esm2_1 import ( + Cl, + Clcalipso, + Cli, + Clw, + Omon, +) +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor.table import get_var_info def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-ESM2-1', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -27,7 +32,7 @@ def test_cl_fix(): def test_get_clcalipso_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-ESM2-1', 'Amon', 'clcalipso') - assert fix == [Clcalipso(None)] + assert fix == [Clcalipso(None), GenericFix(None)] def test_clcalipso_fix(): @@ -38,7 +43,7 @@ def test_clcalipso_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-ESM2-1', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -49,7 +54,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-ESM2-1', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -88,7 +93,7 @@ def thetao_cubes(): def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CNRM-ESM2-1', 'Omon', 'thetao') - assert fix == [Omon(None)] + assert fix == [Omon(None), GenericFix(None)] def test_thetao_fix_metadata(thetao_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_e3sm_1_0.py b/tests/integration/cmor/_fixes/cmip6/test_e3sm_1_0.py index e86de9e3a2..890c81fd6b 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_e3sm_1_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_e3sm_1_0.py @@ -1,13 +1,14 @@ """Tests for the fixes of E3SM-1-0.""" from esmvalcore.cmor._fixes.cmip6.e3sm_1_0 import Cl from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'E3SM-1-0', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg.py b/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg.py index f30904b46e..b9145e6e87 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg.py @@ -14,6 +14,7 @@ Siconca, Tas, ) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -30,7 +31,7 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'SImon', 'siconca'), - [Siconca(None)]) + [Siconca(None), GenericFix(None)]) def test_fix_data(self): """Test data fix.""" @@ -41,13 +42,13 @@ def test_fix_data(self): def test_get_siconc_fix(): """Test sinconc calendar is fixed.""" - fix, = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'SImon', 'siconc') + fix = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'SImon', 'siconc')[0] assert isinstance(fix, CalendarFix) def test_get_tos_fix(): """Test tos calendar is fixed.""" - fix, = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'Omon', 'tos') + fix = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'Omon', 'tos')[0] assert isinstance(fix, CalendarFix) @@ -110,7 +111,7 @@ def tas_cubes(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix_metadata(tas_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg_lr.py b/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg_lr.py index 22abcc7d33..b3f7963a7d 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ec_earth3_veg_lr.py @@ -1,13 +1,13 @@ """Test fixes for EC-Earth3-Veg-LR.""" from esmvalcore.cmor._fixes.cmip6.ec_earth3_veg_lr import Siconc from esmvalcore.cmor._fixes.common import OceanFixGrid -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'EC-Earth3-Veg-LR', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_fgoals_f3_l.py b/tests/integration/cmor/_fixes/cmip6/test_fgoals_f3_l.py index 450158ca7d..7e9aa38d06 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_fgoals_f3_l.py +++ b/tests/integration/cmor/_fixes/cmip6/test_fgoals_f3_l.py @@ -1,17 +1,13 @@ """Tests for the fixes of FGOALS-f3-L.""" -import numpy as np import iris +import numpy as np import pytest from cf_units import Unit -from esmvalcore.cmor._fixes.cmip6.fgoals_f3_l import ( - AllVars, - Clt, - Sftlf, - Tos, -) +from esmvalcore.cmor._fixes.cmip6.fgoals_f3_l import AllVars, Clt, Sftlf, Tos from esmvalcore.cmor._fixes.common import OceanFixGrid +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -80,7 +76,7 @@ def cubes(): def test_get_allvars_fix(): fix = Fix.get_fixes('CMIP6', 'FGOALS-f3-L', 'Amon', 'wrong_time_bnds') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(cubes): @@ -104,7 +100,7 @@ def test_tos_fix(): def test_get_clt_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FGOALS-f3-l', 'Amon', 'clt') - assert fix == [Clt(None), AllVars(None)] + assert fix == [Clt(None), AllVars(None), GenericFix(None)] @pytest.fixture @@ -129,7 +125,7 @@ def test_clt_fix_data(clt_cube): def test_get_sftlf_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FGOALS-f3-l', 'Amon', 'sftlf') - assert fix == [Sftlf(None), AllVars(None)] + assert fix == [Sftlf(None), AllVars(None), GenericFix(None)] def test_sftlf_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py b/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py index 630aa99fca..8bb9457021 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py +++ b/tests/integration/cmor/_fixes/cmip6/test_fgoals_g3.py @@ -6,6 +6,7 @@ from esmvalcore.cmor._fixes.cmip6.fgoals_g3 import Mrsos, Siconc, Tos from esmvalcore.cmor._fixes.common import OceanFixGrid +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -13,7 +14,7 @@ def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Omon', 'tos') - assert fix == [Tos(None)] + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix(): @@ -56,7 +57,7 @@ def test_tos_fix_metadata(mock_base_fix_metadata): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -67,7 +68,7 @@ def test_siconc_fix(): def test_get_mrsos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FGOALS-g3', 'Lmon', 'mrsos') - assert fix == [Mrsos(None)] + assert fix == [Mrsos(None), GenericFix(None)] def test_mrsos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py b/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py index 36696f122b..4ac1d05d4c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_fio_esm_2_0.py @@ -6,6 +6,7 @@ from esmvalcore.cmor._fixes.cmip6.fio_esm_2_0 import Amon, Omon, Tos from esmvalcore.cmor._fixes.common import OceanFixGrid +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -13,13 +14,13 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FIO-ESM-2-0', 'Amon', 'tas') - assert fix == [Amon(None)] + assert fix == [Amon(None), GenericFix(None)] def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'FIO-ESM-2-0', 'Omon', 'tos') - assert fix == [OceanFixGrid(None), Omon(None)] + assert fix == [OceanFixGrid(None), Omon(None), GenericFix(None)] def test_tos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py index e52b55a001..f28310c6c2 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py @@ -3,9 +3,16 @@ import numpy as np import pytest -from esmvalcore.cmor._fixes.cmip6.gfdl_cm4 import (Cl, Cli, Clw, - Fgco2, Omon, Siconc) +from esmvalcore.cmor._fixes.cmip6.gfdl_cm4 import ( + Cl, + Cli, + Clw, + Fgco2, + Omon, + Siconc, +) from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -13,7 +20,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] AIR_PRESSURE_POINTS = np.array([[[[1.0, 1.0], @@ -75,7 +82,7 @@ def test_cl_fix_metadata(test_data_path): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -86,7 +93,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -97,13 +104,13 @@ def test_clw_fix(): def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-CM4', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py index d21323bb10..1db8eca4a0 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py @@ -6,6 +6,7 @@ from esmvalcore.cmor._fixes.cmip6.gfdl_esm4 import Fgco2, Omon, Siconc from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -13,7 +14,7 @@ def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-ESM4', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -52,7 +53,7 @@ def thetao_cubes(): def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-ESM4', 'Omon', 'thetao') - assert fix == [Omon(None)] + assert fix == [Omon(None), GenericFix(None)] def test_thetao_fix_metadata(thetao_cubes): @@ -76,7 +77,7 @@ def test_thetao_fix_metadata(thetao_cubes): def test_get_fgco2_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GFDL-ESM4', 'Omon', 'fgco2') - assert fix == [Fgco2(None), Omon(None)] + assert fix == [Fgco2(None), Omon(None), GenericFix(None)] def test_fgco2_fix_metadata(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_g.py b/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_g.py index 9c88dc61aa..40e61adb86 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_g.py +++ b/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_g.py @@ -5,13 +5,13 @@ from esmvalcore.cmor._fixes.cmip6.giss_e2_1_g import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-G', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -22,7 +22,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-G', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -33,7 +33,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-G', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -42,7 +42,7 @@ def test_clw_fix(): def test_tos_fix(): - fix, = Fix.get_fixes('CMIP6', 'GISS-E2-1-G', 'Omon', 'tos') + fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-G', 'Omon', 'tos')[0] cube = Cube( da.array([274], dtype=np.float32), var_name='tos', diff --git a/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_h.py b/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_h.py index f8afbeea0a..482e72cd22 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_h.py +++ b/tests/integration/cmor/_fixes/cmip6/test_giss_e2_1_h.py @@ -1,13 +1,13 @@ """Test fixes for GISS-E2-1-H.""" from esmvalcore.cmor._fixes.cmip6.giss_e2_1_h import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-H', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +18,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-H', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +29,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'GISS-E2-1-H', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py index a6543060be..aa9e7c34cb 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py +++ b/tests/integration/cmor/_fixes/cmip6/test_hadgem3_gc31_ll.py @@ -4,6 +4,7 @@ from esmvalcore.cmor._fixes.cmip6.hadgem3_gc31_ll import AllVars, Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -16,7 +17,7 @@ def sample_cubes(): def test_get_tas_fix(): fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(sample_cubes): @@ -40,7 +41,7 @@ def test_allvars_no_need_tofix_metadata(sample_cubes): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): @@ -51,7 +52,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'cli') - assert fix == [Cli(None), AllVars(None)] + assert fix == [Cli(None), AllVars(None), GenericFix(None)] def test_cli_fix(): @@ -62,7 +63,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'HadGEM3-GC31-LL', 'Amon', 'clw') - assert fix == [Clw(None), AllVars(None)] + assert fix == [Clw(None), AllVars(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_icon_esm_lr.py b/tests/integration/cmor/_fixes/cmip6/test_icon_esm_lr.py index 705a8bb5ef..dfd183788d 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_icon_esm_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_icon_esm_lr.py @@ -4,6 +4,7 @@ from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.cmip6.icon_esm_lr import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -34,7 +35,7 @@ def cubes(): def test_get_allvars_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'ICON-ESM-LR', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata_lat_lon(cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_iitm_esm.py b/tests/integration/cmor/_fixes/cmip6/test_iitm_esm.py index 52380e87fd..16efd1d75c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_iitm_esm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_iitm_esm.py @@ -6,6 +6,7 @@ from esmvalcore.cmor._fixes.cmip6.iitm_esm import AllVars, Tos from esmvalcore.cmor._fixes.common import OceanFixGrid +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -16,7 +17,7 @@ def test_get_tos_fix(): 'Omon', 'tos', extra_facets={"frequency": "mon"}) - assert fix == [Tos(None), AllVars(None)] + assert fix == [Tos(None), AllVars(None), GenericFix(None)] def test_tos_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py index 99f2a76aa2..c32008ef0c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py @@ -9,7 +9,7 @@ from iris.exceptions import CoordinateNotFoundError from esmvalcore.cmor._fixes.cmip6.ipsl_cm6a_lr import AllVars, Clcalipso, Omon -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor.table import get_var_info @@ -79,7 +79,7 @@ def test_fix_data_no_lat_lon(self): def test_get_clcalipso_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'IPSL-CM6A-LR', 'CFmon', 'clcalipso') - assert fix == [Clcalipso(None), AllVars(None)] + assert fix == [Clcalipso(None), AllVars(None), GenericFix(None)] @pytest.fixture @@ -137,7 +137,7 @@ def thetao_cubes(): def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'IPSL-CM6A-LR', 'Omon', 'thetao') - assert fix == [Omon(None), AllVars(None)] + assert fix == [Omon(None), AllVars(None), GenericFix(None)] def test_thetao_fix_metadata(thetao_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_kace_1_0_g.py b/tests/integration/cmor/_fixes/cmip6/test_kace_1_0_g.py index 21e35f0c01..5d6b4641d8 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_kace_1_0_g.py +++ b/tests/integration/cmor/_fixes/cmip6/test_kace_1_0_g.py @@ -6,13 +6,14 @@ from esmvalcore.cmor._fixes.cmip6.kace_1_0_g import AllVars, Cl, Cli, Clw, Tos from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord, OceanFixGrid +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'KACE-1-0-G', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): @@ -23,7 +24,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'KACE-1-0-G', 'Amon', 'cli') - assert fix == [Cli(None), AllVars(None)] + assert fix == [Cli(None), AllVars(None), GenericFix(None)] def test_cli_fix(): @@ -34,7 +35,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'KACE-1-0-G', 'Amon', 'clw') - assert fix == [Clw(None), AllVars(None)] + assert fix == [Clw(None), AllVars(None), GenericFix(None)] def test_clw_fix(): @@ -45,7 +46,7 @@ def test_clw_fix(): def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'KACE-1-0-G', 'Omon', 'tos') - assert fix == [Tos(None), AllVars(None)] + assert fix == [Tos(None), AllVars(None), GenericFix(None)] def test_tos_fix(): @@ -101,7 +102,7 @@ def tos_cubes(): def test_get_allvars_fix(): fix = Fix.get_fixes('CMIP6', 'KACE-1-0-G', 'Omon', 'tos') - assert fix == [OceanFixGrid(None), AllVars(None)] + assert fix == [OceanFixGrid(None), AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(monkeypatch, tos_cubes, caplog): diff --git a/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py b/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py index 9b460db15a..80af6436e1 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_kiost_esm.py @@ -8,7 +8,7 @@ from esmvalcore.cmor._fixes.cmip6.kiost_esm import SfcWind, Siconc, Tas from esmvalcore.cmor._fixes.common import SiconcFixScalarCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor.table import get_var_info @@ -72,7 +72,7 @@ def tas_cubes(): def test_get_sfcwind_fix(): fix = Fix.get_fixes('CMIP6', 'KIOST-ESM', 'Amon', 'sfcWind') - assert fix == [SfcWind(None)] + assert fix == [SfcWind(None), GenericFix(None)] def test_sfcwind_fix_metadata(sfcwind_cubes): @@ -104,7 +104,7 @@ def test_sfcwind_fix_metadata(sfcwind_cubes): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'KIOST-ESM', 'SImon', 'siconc') - assert fix == [Siconc(None)] + assert fix == [Siconc(None), GenericFix(None)] def test_siconc_fix(): @@ -127,7 +127,7 @@ def test_siconc_fix_data(): def test_get_tas_fix(): fix = Fix.get_fixes('CMIP6', 'KIOST-ESM', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] def test_tas_fix_metadata(tas_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py index bbb1a98f32..85eed25338 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py @@ -5,6 +5,7 @@ from cf_units import Unit from esmvalcore.cmor._fixes.cmip6.mcm_ua_1_0 import AllVars, Omon, Tas, Uas +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -119,17 +120,17 @@ def cubes_bounds(): def test_get_allvars_fix(): fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'arbitrary_var_name_and_wrong_lon_bnds') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_get_tas_fix(): fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'tas') - assert fix == [Tas(None), AllVars(None)] + assert fix == [Tas(None), AllVars(None), GenericFix(None)] def test_get_uas_fix(): fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Amon', 'uas') - assert fix == [Uas(None), AllVars(None)] + assert fix == [Uas(None), AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(cubes): @@ -265,7 +266,7 @@ def thetao_cubes(): def test_get_thetao_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MCM-UA-1-0', 'Omon', 'thetao') - assert fix == [Omon(None), AllVars(None)] + assert fix == [Omon(None), AllVars(None), GenericFix(None)] def test_thetao_fix_metadata(thetao_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_miroc6.py b/tests/integration/cmor/_fixes/cmip6/test_miroc6.py index 9c021d9ce5..26662a0ec3 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_miroc6.py +++ b/tests/integration/cmor/_fixes/cmip6/test_miroc6.py @@ -5,14 +5,14 @@ from esmvalcore.cmor._fixes.cmip6.miroc6 import Cl, Cli, Clw, Tos from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor.table import get_var_info def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC6', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -23,7 +23,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC6', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -34,7 +34,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC6', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -69,7 +69,7 @@ def tos_cubes(): def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC6', 'Omon', 'tos') - assert fix == [Tos(None)] + assert fix == [Tos(None), GenericFix(None)] def test_tos_fix_metadata(tos_cubes): diff --git a/tests/integration/cmor/_fixes/cmip6/test_miroc_es2l.py b/tests/integration/cmor/_fixes/cmip6/test_miroc_es2l.py index 550f01d80c..22ae2d1c03 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_miroc_es2l.py +++ b/tests/integration/cmor/_fixes/cmip6/test_miroc_es2l.py @@ -1,13 +1,13 @@ """Test fixes for MIROC-ES2L.""" from esmvalcore.cmor._fixes.cmip6.miroc_es2l import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC-ES2L', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +18,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC-ES2L', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +29,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MIROC-ES2L', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_lr.py b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_lr.py index 0f05c763cb..2ae3af9355 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm1_2_lr.py @@ -1,13 +1,14 @@ """Tests for the fixes of MPI-ESM1-2-LR.""" from esmvalcore.cmor._fixes.cmip6.mpi_esm1_2_lr import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM1-2-LR', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +19,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM1-2-LR', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +30,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM1-2-LR', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm_1_2_ham.py b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm_1_2_ham.py index 492aa45f57..358cb1f56f 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mpi_esm_1_2_ham.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mpi_esm_1_2_ham.py @@ -1,13 +1,14 @@ """Tests for the fixes of MPI-ESM-1-2-HAM.""" from esmvalcore.cmor._fixes.cmip6.mpi_esm_1_2_ham import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM-1-2-HAM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +19,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM-1-2-HAM', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +30,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MPI-ESM-1-2-HAM', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_mri_esm2_0.py b/tests/integration/cmor/_fixes/cmip6/test_mri_esm2_0.py index 609cd12abc..6ec68855b7 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mri_esm2_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mri_esm2_0.py @@ -1,13 +1,13 @@ """Test fixes for MRI-ESM2-0.""" from esmvalcore.cmor._fixes.cmip6.mri_esm2_0 import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MRI-ESM2-0', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +18,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MRI-ESM2-0', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +29,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'MRI-ESM2-0', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_nesm3.py b/tests/integration/cmor/_fixes/cmip6/test_nesm3.py index c1e1fe5f94..bc9265bc5b 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_nesm3.py +++ b/tests/integration/cmor/_fixes/cmip6/test_nesm3.py @@ -1,13 +1,13 @@ """Test fixes for NESM3.""" from esmvalcore.cmor._fixes.cmip6.nesm3 import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NESM3', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +18,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NESM3', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +29,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NESM3', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py b/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py index c77e945dee..78ea66effc 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py @@ -12,6 +12,7 @@ Siconc, ) from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -19,7 +20,7 @@ def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-LM', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): @@ -30,7 +31,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-LM', 'Amon', 'cli') - assert fix == [Cli(None), AllVars(None)] + assert fix == [Cli(None), AllVars(None), GenericFix(None)] def test_cli_fix(): @@ -41,7 +42,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-LM', 'Amon', 'clw') - assert fix == [Clw(None), AllVars(None)] + assert fix == [Clw(None), AllVars(None), GenericFix(None)] def test_clw_fix(): @@ -97,7 +98,7 @@ def cubes_bounds(): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-LM', 'SImon', 'siconc') - assert fix == [Siconc(None), AllVars(None)] + assert fix == [Siconc(None), AllVars(None), GenericFix(None)] def test_allvars_fix_lon_bounds(cubes_bounds): diff --git a/tests/integration/cmor/_fixes/cmip6/test_noresm2_mm.py b/tests/integration/cmor/_fixes/cmip6/test_noresm2_mm.py index 2fd3a9c7f4..0868dd910a 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_noresm2_mm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_noresm2_mm.py @@ -1,13 +1,14 @@ """Tests for the fixes of NorESM2-MM.""" from esmvalcore.cmor._fixes.cmip6.noresm2_mm import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-MM', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +19,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-MM', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +30,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'NorESM2-MM', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_sam0_unicon.py b/tests/integration/cmor/_fixes/cmip6/test_sam0_unicon.py index 09d35be726..e9f0a76a50 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_sam0_unicon.py +++ b/tests/integration/cmor/_fixes/cmip6/test_sam0_unicon.py @@ -5,13 +5,13 @@ from esmvalcore.cmor._fixes.cmip6.sam0_unicon import Cl, Cli, Clw, Nbp from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord -from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.fix import Fix, GenericFix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'SAM0-UNICON', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -22,7 +22,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'SAM0-UNICON', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -33,7 +33,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'SAM0-UNICON', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): @@ -44,7 +44,7 @@ def test_clw_fix(): def test_get_nbp_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'SAM0-UNICON', 'Lmon', 'nbp') - assert fix == [Nbp(None)] + assert fix == [Nbp(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/cmip6/test_taiesm1.py b/tests/integration/cmor/_fixes/cmip6/test_taiesm1.py index 107db534bd..dd955ef1fb 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_taiesm1.py +++ b/tests/integration/cmor/_fixes/cmip6/test_taiesm1.py @@ -1,13 +1,14 @@ """Tests for the fixes of TaiESM1.""" from esmvalcore.cmor._fixes.cmip6.taiesm1 import Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'TaiESM1', 'Amon', 'cl') - assert fix == [Cl(None)] + assert fix == [Cl(None), GenericFix(None)] def test_cl_fix(): @@ -18,7 +19,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'TaiESM1', 'Amon', 'cli') - assert fix == [Cli(None)] + assert fix == [Cli(None), GenericFix(None)] def test_cli_fix(): @@ -29,7 +30,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'TaiESM1', 'Amon', 'clw') - assert fix == [Clw(None)] + assert fix == [Clw(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py index f6aceb1211..423d55e637 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ukesm1_0_ll.py @@ -4,6 +4,7 @@ from esmvalcore.cmor._fixes.cmip6.ukesm1_0_ll import AllVars, Cl, Cli, Clw from esmvalcore.cmor._fixes.common import ClFixHybridHeightCoord +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix @@ -18,7 +19,7 @@ def sample_cubes(): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_metadata(sample_cubes): @@ -44,7 +45,7 @@ def test_allvars_no_need_tofix_metadata(sample_cubes): def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'cl') - assert fix == [Cl(None), AllVars(None)] + assert fix == [Cl(None), AllVars(None), GenericFix(None)] def test_cl_fix(): @@ -55,7 +56,7 @@ def test_cl_fix(): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'cli') - assert fix == [Cli(None), AllVars(None)] + assert fix == [Cli(None), AllVars(None), GenericFix(None)] def test_cli_fix(): @@ -66,7 +67,7 @@ def test_cli_fix(): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'UKESM1-0-LL', 'Amon', 'clw') - assert fix == [Clw(None), AllVars(None)] + assert fix == [Clw(None), AllVars(None), GenericFix(None)] def test_clw_fix(): diff --git a/tests/integration/cmor/_fixes/emac/test_emac.py b/tests/integration/cmor/_fixes/emac/test_emac.py index 15c5b76a6a..1146502a42 100644 --- a/tests/integration/cmor/_fixes/emac/test_emac.py +++ b/tests/integration/cmor/_fixes/emac/test_emac.py @@ -36,6 +36,7 @@ Toz, Zg, ) +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import CoordinateInfo, get_var_info from esmvalcore.config._config import get_extra_facets @@ -834,7 +835,7 @@ def test_sample_data_ta_alevel(test_data_path, tmp_path): def test_get_awhea_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Omon', 'awhea') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_awhea_fix(cubes_2d): @@ -859,7 +860,7 @@ def test_awhea_fix(cubes_2d): def test_get_clivi_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'clivi') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_clivi_fix(cubes_2d): @@ -883,7 +884,7 @@ def test_clivi_fix(cubes_2d): def test_get_clt_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'clt') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_clt_fix(cubes_2d): @@ -906,7 +907,7 @@ def test_clt_fix(cubes_2d): def test_get_clwvi_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'clwvi') - assert fix == [Clwvi(None), AllVars(None)] + assert fix == [Clwvi(None), AllVars(None), GenericFix(None)] def test_clwvi_fix(cubes_2d): @@ -933,7 +934,7 @@ def test_clwvi_fix(cubes_2d): def test_get_co2mass_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'co2mass') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_co2mass_fix(cubes_1d): @@ -957,7 +958,7 @@ def test_co2mass_fix(cubes_1d): def test_get_evspsbl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'evspsbl') - assert fix == [Evspsbl(None), AllVars(None)] + assert fix == [Evspsbl(None), AllVars(None), GenericFix(None)] def test_evspsbl_fix(cubes_2d): @@ -986,7 +987,7 @@ def test_evspsbl_fix(cubes_2d): def test_get_hfls_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'hfls') - assert fix == [Hfls(None), AllVars(None)] + assert fix == [Hfls(None), AllVars(None), GenericFix(None)] def test_hfls_fix(cubes_2d): @@ -1014,7 +1015,7 @@ def test_hfls_fix(cubes_2d): def test_get_hfss_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'hfss') - assert fix == [Hfss(None), AllVars(None)] + assert fix == [Hfss(None), AllVars(None), GenericFix(None)] def test_hfss_fix(cubes_2d): @@ -1042,7 +1043,7 @@ def test_hfss_fix(cubes_2d): def test_get_hurs_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'hurs') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_hurs_fix(cubes_2d): @@ -1065,7 +1066,7 @@ def test_hurs_fix(cubes_2d): def test_get_od550aer_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'od550aer') - assert fix == [Od550aer(None), AllVars(None)] + assert fix == [Od550aer(None), AllVars(None), GenericFix(None)] def test_od550aer_fix(cubes_3d): @@ -1090,7 +1091,7 @@ def test_od550aer_fix(cubes_3d): def test_get_pr_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'pr') - assert fix == [Pr(None), AllVars(None)] + assert fix == [Pr(None), AllVars(None), GenericFix(None)] def test_pr_fix(cubes_2d): @@ -1116,7 +1117,7 @@ def test_pr_fix(cubes_2d): def test_get_prc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'prc') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_prc_fix(cubes_2d): @@ -1140,7 +1141,7 @@ def test_prc_fix(cubes_2d): def test_get_prl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'prl') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_prl_fix(cubes_2d): @@ -1164,7 +1165,7 @@ def test_prl_fix(cubes_2d): def test_get_prsn_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'prsn') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_prsn_fix(cubes_2d): @@ -1188,7 +1189,7 @@ def test_prsn_fix(cubes_2d): def test_get_prw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'prw') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_prw_fix(cubes_2d): @@ -1212,7 +1213,7 @@ def test_prw_fix(cubes_2d): def test_get_ps_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'ps') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ps_fix(cubes_2d): @@ -1236,7 +1237,7 @@ def test_ps_fix(cubes_2d): def test_get_psl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'psl') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_psl_fix(cubes_2d): @@ -1260,7 +1261,7 @@ def test_psl_fix(cubes_2d): def test_get_rlds_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rlds') - assert fix == [Rlds(None), AllVars(None)] + assert fix == [Rlds(None), AllVars(None), GenericFix(None)] def test_rlds_fix(cubes_2d): @@ -1285,7 +1286,7 @@ def test_rlds_fix(cubes_2d): def test_get_rlus_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rlus') - assert fix == [Rlus(None), AllVars(None)] + assert fix == [Rlus(None), AllVars(None), GenericFix(None)] def test_rlus_fix(cubes_2d): @@ -1313,7 +1314,7 @@ def test_rlus_fix(cubes_2d): def test_get_rlut_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rlut') - assert fix == [Rlut(None), AllVars(None)] + assert fix == [Rlut(None), AllVars(None), GenericFix(None)] def test_rlut_fix(cubes_2d): @@ -1341,7 +1342,7 @@ def test_rlut_fix(cubes_2d): def test_get_rlutcs_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rlutcs') - assert fix == [Rlutcs(None), AllVars(None)] + assert fix == [Rlutcs(None), AllVars(None), GenericFix(None)] def test_rlutcs_fix(cubes_2d): @@ -1370,7 +1371,7 @@ def test_rlutcs_fix(cubes_2d): def test_get_rsds_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rsds') - assert fix == [Rsds(None), AllVars(None)] + assert fix == [Rsds(None), AllVars(None), GenericFix(None)] def test_rsds_fix(cubes_2d): @@ -1395,7 +1396,7 @@ def test_rsds_fix(cubes_2d): def test_get_rsdt_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rsdt') - assert fix == [Rsdt(None), AllVars(None)] + assert fix == [Rsdt(None), AllVars(None), GenericFix(None)] def test_rsdt_fix(cubes_2d): @@ -1420,7 +1421,7 @@ def test_rsdt_fix(cubes_2d): def test_get_rsus_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rsus') - assert fix == [Rsus(None), AllVars(None)] + assert fix == [Rsus(None), AllVars(None), GenericFix(None)] def test_rsus_fix(cubes_2d): @@ -1448,7 +1449,7 @@ def test_rsus_fix(cubes_2d): def test_get_rsut_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rsut') - assert fix == [Rsut(None), AllVars(None)] + assert fix == [Rsut(None), AllVars(None), GenericFix(None)] def test_rsut_fix(cubes_2d): @@ -1476,7 +1477,7 @@ def test_rsut_fix(cubes_2d): def test_get_rsutcs_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rsutcs') - assert fix == [Rsutcs(None), AllVars(None)] + assert fix == [Rsutcs(None), AllVars(None), GenericFix(None)] def test_rsutcs_fix(cubes_2d): @@ -1505,7 +1506,7 @@ def test_rsutcs_fix(cubes_2d): def test_get_rtmt_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'rtmt') - assert fix == [Rtmt(None), AllVars(None)] + assert fix == [Rtmt(None), AllVars(None), GenericFix(None)] def test_rtmt_fix(cubes_2d): @@ -1531,7 +1532,7 @@ def test_rtmt_fix(cubes_2d): def test_get_sfcWind_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'sfcWind') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_sfcWind_fix(cubes_2d): # noqa: N802 @@ -1557,7 +1558,7 @@ def test_sfcWind_fix(cubes_2d): # noqa: N802 def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'SImon', 'siconc') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_siconc_fix(cubes_2d): @@ -1582,7 +1583,7 @@ def test_siconc_fix(cubes_2d): def test_get_siconca_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'SImon', 'siconca') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_siconca_fix(cubes_2d): @@ -1607,7 +1608,7 @@ def test_siconca_fix(cubes_2d): def test_get_sithick_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'SImon', 'sithick') - assert fix == [Sithick(None), AllVars(None)] + assert fix == [Sithick(None), AllVars(None), GenericFix(None)] def test_sithick_fix(cubes_2d): @@ -1642,7 +1643,7 @@ def test_sithick_fix(cubes_2d): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tas_fix(cubes_2d): @@ -1668,7 +1669,7 @@ def test_tas_fix(cubes_2d): def test_get_tasmax_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'tasmax') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tasmax_fix(cubes_2d): @@ -1694,7 +1695,7 @@ def test_tasmax_fix(cubes_2d): def test_get_tasmin_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'tasmin') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tasmin_fix(cubes_2d): @@ -1720,7 +1721,7 @@ def test_tasmin_fix(cubes_2d): def test_get_tauu_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'tauu') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tauu_fix(cubes_2d): @@ -1744,7 +1745,7 @@ def test_tauu_fix(cubes_2d): def test_get_tauv_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'tauv') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tauv_fix(cubes_2d): @@ -1768,7 +1769,7 @@ def test_tauv_fix(cubes_2d): def test_get_tos_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Omon', 'tos') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tos_fix(cubes_2d): @@ -1792,7 +1793,7 @@ def test_tos_fix(cubes_2d): def test_get_toz_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'AERmon', 'toz') - assert fix == [Toz(None), AllVars(None)] + assert fix == [Toz(None), AllVars(None), GenericFix(None)] def test_toz_fix(cubes_2d): @@ -1816,7 +1817,7 @@ def test_toz_fix(cubes_2d): def test_get_ts_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'ts') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ts_fix(cubes_2d): @@ -1840,7 +1841,7 @@ def test_ts_fix(cubes_2d): def test_get_uas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'uas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_uas_fix(cubes_2d): @@ -1866,7 +1867,7 @@ def test_uas_fix(cubes_2d): def test_get_vas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'vas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_vas_fix(cubes_2d): @@ -1895,7 +1896,7 @@ def test_vas_fix(cubes_2d): def test_get_MP_BC_tot_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_BC_tot') - assert fix == [MP_BC_tot(None), AllVars(None)] + assert fix == [MP_BC_tot(None), AllVars(None), GenericFix(None)] def test_MP_BC_tot_fix(cubes_1d): # noqa: N802 @@ -1925,7 +1926,7 @@ def test_MP_BC_tot_fix(cubes_1d): # noqa: N802 def test_get_MP_CFCl3_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_CFCl3') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_CFCl3_fix(cubes_1d): # noqa: N802 @@ -1949,7 +1950,7 @@ def test_MP_CFCl3_fix(cubes_1d): # noqa: N802 def test_get_MP_ClOX_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_ClOX') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_ClOX_fix(cubes_1d): # noqa: N802 @@ -1973,7 +1974,7 @@ def test_MP_ClOX_fix(cubes_1d): # noqa: N802 def test_get_MP_CH4_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_CH4') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_CH4_fix(cubes_1d): # noqa: N802 @@ -1997,7 +1998,7 @@ def test_MP_CH4_fix(cubes_1d): # noqa: N802 def test_get_MP_CO_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_CO') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_CO_fix(cubes_1d): # noqa: N802 @@ -2021,7 +2022,7 @@ def test_MP_CO_fix(cubes_1d): # noqa: N802 def test_get_MP_CO2_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_CO2') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_CO2_fix(cubes_1d): # noqa: N802 @@ -2045,7 +2046,7 @@ def test_MP_CO2_fix(cubes_1d): # noqa: N802 def test_get_MP_DU_tot_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_DU_tot') - assert fix == [MP_DU_tot(None), AllVars(None)] + assert fix == [MP_DU_tot(None), AllVars(None), GenericFix(None)] def test_MP_DU_tot_fix(cubes_1d): # noqa: N802 @@ -2076,7 +2077,7 @@ def test_MP_DU_tot_fix(cubes_1d): # noqa: N802 def test_get_MP_N2O_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_N2O') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_N2O_fix(cubes_1d): # noqa: N802 @@ -2100,7 +2101,7 @@ def test_MP_N2O_fix(cubes_1d): # noqa: N802 def test_get_MP_NH3_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_NH3') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_NH3_fix(cubes_1d): # noqa: N802 @@ -2124,7 +2125,7 @@ def test_MP_NH3_fix(cubes_1d): # noqa: N802 def test_get_MP_NO_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_NO') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_NO_fix(cubes_1d): # noqa: N802 @@ -2148,7 +2149,7 @@ def test_MP_NO_fix(cubes_1d): # noqa: N802 def test_get_MP_NO2_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_NO2') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_NO2_fix(cubes_1d): # noqa: N802 @@ -2172,7 +2173,7 @@ def test_MP_NO2_fix(cubes_1d): # noqa: N802 def test_get_MP_NOX_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_NOX') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_NOX_fix(cubes_1d): # noqa: N802 @@ -2196,7 +2197,7 @@ def test_MP_NOX_fix(cubes_1d): # noqa: N802 def test_get_MP_O3_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_O3') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_O3_fix(cubes_1d): # noqa: N802 @@ -2220,7 +2221,7 @@ def test_MP_O3_fix(cubes_1d): # noqa: N802 def test_get_MP_OH_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_OH') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_OH_fix(cubes_1d): # noqa: N802 @@ -2244,7 +2245,7 @@ def test_MP_OH_fix(cubes_1d): # noqa: N802 def test_get_MP_S_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_S') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_S_fix(cubes_1d): # noqa: N802 @@ -2268,7 +2269,7 @@ def test_MP_S_fix(cubes_1d): # noqa: N802 def test_get_MP_SO2_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_SO2') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_MP_SO2_fix(cubes_1d): # noqa: N802 @@ -2292,7 +2293,7 @@ def test_MP_SO2_fix(cubes_1d): # noqa: N802 def test_get_MP_SO4mm_tot_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_SO4mm_tot') - assert fix == [MP_SO4mm_tot(None), AllVars(None)] + assert fix == [MP_SO4mm_tot(None), AllVars(None), GenericFix(None)] def test_MP_SO4mm_tot_fix(cubes_1d): # noqa: N802 @@ -2323,7 +2324,7 @@ def test_MP_SO4mm_tot_fix(cubes_1d): # noqa: N802 def test_get_MP_SS_tot_fix(): # noqa: N802 """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'TRAC10hr', 'MP_SS_tot') - assert fix == [MP_SS_tot(None), AllVars(None)] + assert fix == [MP_SS_tot(None), AllVars(None), GenericFix(None)] def test_MP_SS_tot_fix(cubes_1d): # noqa: N802 @@ -2355,7 +2356,7 @@ def test_MP_SS_tot_fix(cubes_1d): # noqa: N802 def test_get_cl_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'cl') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_cl_fix(cubes_3d): @@ -2380,7 +2381,7 @@ def test_cl_fix(cubes_3d): def test_get_cli_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'cli') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_cli_fix(cubes_3d): @@ -2406,7 +2407,7 @@ def test_cli_fix(cubes_3d): def test_get_clw_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'clw') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_clw_fix(cubes_3d): @@ -2432,7 +2433,7 @@ def test_clw_fix(cubes_3d): def test_get_hur_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'hur') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_hur_fix(cubes_3d): @@ -2459,7 +2460,7 @@ def test_hur_fix(cubes_3d): def test_get_hus_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'hus') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_hus_fix(cubes_3d): @@ -2486,7 +2487,7 @@ def test_hus_fix(cubes_3d): def test_get_ta_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'ta') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ta_fix(cubes_3d): @@ -2513,7 +2514,7 @@ def test_ta_fix(cubes_3d): def test_get_ua_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'ua') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ua_fix(cubes_3d): @@ -2540,7 +2541,7 @@ def test_ua_fix(cubes_3d): def test_get_va_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'va') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_va_fix(cubes_3d): @@ -2567,7 +2568,7 @@ def test_va_fix(cubes_3d): def test_get_zg_fix(): """Test getting of fix.""" fix = Fix.get_fixes('EMAC', 'EMAC', 'Amon', 'zg') - assert fix == [Zg(None), AllVars(None)] + assert fix == [Zg(None), AllVars(None), GenericFix(None)] def test_zg_fix(cubes_3d): diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index 297058a9a1..be20131e58 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -12,6 +12,7 @@ from iris.cube import Cube, CubeList import esmvalcore.cmor._fixes.icon.icon +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._fixes.icon._base_fixes import IconFix from esmvalcore.cmor._fixes.icon.icon import AllVars, Clwvi from esmvalcore.cmor.fix import Fix @@ -491,7 +492,7 @@ def check_typesi(cube): def test_get_areacella_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'fx', 'areacella') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_areacella_fix(cubes_grid): @@ -513,7 +514,7 @@ def test_areacella_fix(cubes_grid): def test_get_areacello_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Ofx', 'areacello') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_areacello_fix(cubes_grid): @@ -538,7 +539,7 @@ def test_areacello_fix(cubes_grid): def test_get_clwvi_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'clwvi') - assert fix == [Clwvi(None), AllVars(None)] + assert fix == [Clwvi(None), AllVars(None), GenericFix(None)] def test_clwvi_fix(cubes_regular_grid): @@ -572,7 +573,7 @@ def test_clwvi_fix(cubes_regular_grid): def test_get_lwp_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'AERmon', 'lwp') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_lwp_fix(cubes_2d): @@ -599,7 +600,7 @@ def test_lwp_fix(cubes_2d): def test_get_rsdt_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'rsdt') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_rsdt_fix(cubes_2d): @@ -622,7 +623,7 @@ def test_rsdt_fix(cubes_2d): def test_get_rsut_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'rsut') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_rsut_fix(cubes_2d): @@ -648,7 +649,7 @@ def test_rsut_fix(cubes_2d): def test_get_siconc_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'SImon', 'siconc') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_siconc_fix(cubes_2d): @@ -671,7 +672,7 @@ def test_siconc_fix(cubes_2d): def test_get_siconca_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'SImon', 'siconca') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_siconca_fix(cubes_2d): @@ -697,7 +698,7 @@ def test_siconca_fix(cubes_2d): def test_get_ta_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'ta') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ta_fix(cubes_3d): @@ -732,7 +733,7 @@ def test_ta_fix_no_plev_bounds(cubes_3d): def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'tas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_tas_fix(cubes_2d): @@ -854,7 +855,7 @@ def test_tas_no_shift_time(cubes_2d): def test_get_uas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'uas') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_uas_fix(cubes_2d): @@ -951,7 +952,7 @@ def test_2d_lat_lon_grid_fix(cubes_2d_lat_lon_grid): def test_get_ch4clim_fix(): """Test getting of fix.""" fix = Fix.get_fixes('ICON', 'ICON', 'Amon', 'ch4Clim') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_ch4clim_fix(cubes_regular_grid): diff --git a/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py b/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py index f136114bcf..dd1d19480e 100644 --- a/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py +++ b/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py @@ -2,6 +2,7 @@ import iris import pytest +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._fixes.ipslcm.ipsl_cm6 import Tas from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -11,7 +12,7 @@ def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('IPSLCM', 'IPSL-CM6', 'Amon', 'tas') - assert fix == [Tas(None)] + assert fix == [Tas(None), GenericFix(None)] @pytest.fixture diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index cfe20e7fde..6b8a4922b7 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -6,14 +6,16 @@ import pytest from cf_units import Unit +from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor._fixes.native6.era5 import ( AllVars, Evspsbl, Zg, get_frequency, ) -from esmvalcore.cmor.fix import Fix, fix_metadata -from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.cmor.fix import fix_metadata +from esmvalcore.cmor.table import CMOR_TABLES, get_var_info +from esmvalcore.preprocessor import cmor_check_metadata COMMENT = ('Contains modified Copernicus Climate Change Service Information ' f'{datetime.datetime.now().year}') @@ -22,13 +24,15 @@ def test_get_evspsbl_fix(): """Test whether the right fixes are gathered for a single variable.""" fix = Fix.get_fixes('native6', 'ERA5', 'E1hr', 'evspsbl') - assert fix == [Evspsbl(None), AllVars(None)] + vardef = get_var_info('native6', 'E1hr', 'evspsbl') + assert fix == [Evspsbl(vardef), AllVars(vardef), GenericFix(vardef)] def test_get_zg_fix(): """Test whether the right fix gets found again, for zg as well.""" fix = Fix.get_fixes('native6', 'ERA5', 'Amon', 'zg') - assert fix == [Zg(None), AllVars(None)] + vardef = get_var_info('native6', 'E1hr', 'evspsbl') + assert fix == [Zg(vardef), AllVars(vardef), GenericFix(vardef)] def test_get_frequency_hourly(): @@ -1027,9 +1031,14 @@ def uas_cmor_e1hr(): def test_cmorization(era5_cubes, cmor_cubes, var, mip): """Verify that cmorization results in the expected target cube.""" fixed_cubes = fix_metadata(era5_cubes, var, 'native6', 'era5', mip) + assert len(fixed_cubes) == 1 fixed_cube = fixed_cubes[0] cmor_cube = cmor_cubes[0] + + # Test that CMOR checks are passing + fixed_cubes = cmor_check_metadata(fixed_cube, 'native6', mip, var) + if fixed_cube.coords('time'): for cube in [fixed_cube, cmor_cube]: coord = cube.coord('time') diff --git a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_1.py b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_1.py index 4ac2ce7f26..80ce0d00c6 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_1.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_1.py @@ -3,6 +3,7 @@ from iris.coords import DimCoord from iris.cube import Cube, CubeList +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._fixes.obs4mips.airs_2_1 import AllVars from esmvalcore.cmor.fix import Fix @@ -16,7 +17,7 @@ def get_air_pressure_coord(points, units): def test_get_allvars_fix(): """Test getting of fix.""" fix = Fix.get_fixes('obs4MIPs', 'AIRS-2-1', 'Amon', 'cl') - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_allvars_fix_no_air_pressure(): diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py index bde17735c1..6a66486778 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi.py @@ -1,6 +1,7 @@ """Test SSMI fixes.""" import unittest +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._fixes.obs4mips.ssmi import Prw from esmvalcore.cmor.fix import Fix @@ -10,4 +11,4 @@ class TestPrw(unittest.TestCase): def test_get(self): """Test fix get.""" self.assertListEqual(Fix.get_fixes('obs4MIPs', 'SSMI', 'Amon', 'prw'), - [Prw(None)]) + [Prw(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py index 47ec98b12e..024aa4e705 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_ssmi_meris.py @@ -1,6 +1,7 @@ """Test SSMI fixes.""" import unittest +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._fixes.obs4mips.ssmi_meris import Prw from esmvalcore.cmor.fix import Fix @@ -11,4 +12,4 @@ def test_get(self): """Test fix get.""" self.assertListEqual( Fix.get_fixes('obs4MIPs', 'SSMI-MERIS', 'Amon', 'prw'), - [Prw(None)]) + [Prw(None), GenericFix(None)]) diff --git a/tests/integration/cmor/_fixes/test_fix.py b/tests/integration/cmor/_fixes/test_fix.py index 7abc734182..ea4ecec145 100644 --- a/tests/integration/cmor/_fixes/test_fix.py +++ b/tests/integration/cmor/_fixes/test_fix.py @@ -14,16 +14,22 @@ Tas, ) from esmvalcore.cmor._fixes.cordex.cordex_fixes import AllVars +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info from esmvalcore.config import CFG def test_get_fix(): - assert Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgco2') == [FgCo2(None)] + assert Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgco2') == [ + FgCo2(None), GenericFix(None) + ] def test_get_fix_case_insensitive(): - assert Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2'), [FgCo2(None)] + assert Fix.get_fixes('CMIP5', 'CanESM2', 'Amon', 'fgCo2') == [ + FgCo2(None), GenericFix(None) + ] def test_get_fix_cordex(): @@ -34,7 +40,7 @@ def test_get_fix_cordex(): 'tas', extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}, ) - assert fix == [Tas(None), AllVars(None)] + assert fix == [Tas(None), AllVars(None), GenericFix(None)] def test_get_grid_fix_cordex(): @@ -45,15 +51,19 @@ def test_get_grid_fix_cordex(): 'tas', extra_facets={'driver': 'CNRM-CERFACS-CNRM-CM5'}, ) - assert fix == [AllVars(None)] + assert fix == [AllVars(None), GenericFix(None)] def test_get_fixes_with_replace(): - assert Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4') == [Ch4(None)] + assert Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4') == [ + Ch4(None), GenericFix(None) + ] def test_get_fixes_with_generic(): - assert Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp') == [Gpp(None)] + assert Fix.get_fixes('CMIP5', 'CESM1-BGC', 'Amon', 'gpp') == [ + Gpp(None), GenericFix(None) + ] def test_get_fix_no_project(): @@ -62,24 +72,33 @@ def test_get_fix_no_project(): def test_get_fix_no_model(): - assert Fix.get_fixes('CMIP5', 'BAD_MODEL', 'Amon', 'ch4') == [] + assert Fix.get_fixes('CMIP5', 'BAD_MODEL', 'Amon', 'ch4') == [ + GenericFix(None) + ] def test_get_fix_no_var(): - assert Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'BAD_VAR') == [] + assert Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'BAD_VAR') == [ + GenericFix(None) + ] def test_get_fix_only_mip(): - assert Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'thetao') == [Omon(None)] + assert Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'thetao') == [ + Omon(None), GenericFix(None) + ] def test_get_fix_only_mip_case_insensitive(): - assert Fix.get_fixes('CMIP6', 'CESM2', 'omOn', 'thetao') == [Omon(None)] + assert Fix.get_fixes('CMIP6', 'CESM2', 'omOn', 'thetao') == [ + Omon(None), GenericFix(None) + ] def test_get_fix_mip_and_var(): - assert (Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'tos') == - [Tos(None), Omon(None)]) + assert Fix.get_fixes('CMIP6', 'CESM2', 'Omon', 'tos') == [ + Tos(None), Omon(None), GenericFix(None) + ] def test_fix_metadata(): @@ -156,3 +175,25 @@ def test_session(): session = CFG.start_session('my session') fix = Fix(None, session=session) assert fix.session == session + + +def test_frequency_empty(): + fix = Fix(None) + assert fix.frequency is None + + +def test_frequency_from_vardef(): + vardef = get_var_info('CMIP6', 'Amon', 'tas') + fix = Fix(vardef) + assert fix.frequency == 'mon' + + +def test_frequency_given(): + fix = Fix(None, frequency='1hr') + assert fix.frequency == '1hr' + + +def test_frequency_not_from_vardef(): + vardef = get_var_info('CMIP6', 'Amon', 'tas') + fix = Fix(vardef, frequency='3hr') + assert fix.frequency == '3hr' diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py new file mode 100644 index 0000000000..e52dc0fd42 --- /dev/null +++ b/tests/integration/cmor/test_fix.py @@ -0,0 +1,909 @@ +"""Integration tests for fixes.""" + +import dask.array as da +import numpy as np +import pytest +from cf_units import Unit +from iris.aux_factory import HybridHeightFactory, HybridPressureFactory +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube, CubeList + +from esmvalcore.cmor.check import CheckLevels, CMORCheckError +from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.preprocessor import ( + cmor_check_data, + cmor_check_metadata, + fix_data, + fix_metadata, +) + + +# TODO: remove in v2.12 +@pytest.fixture(autouse=True) +def disable_fix_cmor_checker(mocker): + """Disable the CMOR checker in fixes (will be default in v2.12).""" + class MockChecker: + + def __init__(self, cube): + self._cube = cube + + def check_metadata(self): + return self._cube + + def check_data(self): + return self._cube + + mock = mocker.patch('esmvalcore.cmor.fix._get_cmor_checker') + mock.return_value = MockChecker + + +class TestGenericFix: + """Tests for ``GenericFix``.""" + + @pytest.fixture(autouse=True) + def setup(self, mocker): + """Setup tests.""" + self.mock_debug = mocker.patch( + 'esmvalcore.cmor._fixes.fix.GenericFix._debug_msg', autospec=True + ) + self.mock_warning = mocker.patch( + 'esmvalcore.cmor._fixes.fix.GenericFix._warning_msg', + autospec=True, + ) + + # Create sample data with CMOR errors + time_coord = DimCoord( + [15, 45], + standard_name='time', + var_name='time', + units=Unit('days since 1851-01-01', calendar='noleap'), + attributes={'test': 1, 'time_origin': 'will_be_removed'}, + ) + plev_coord_rev = DimCoord( + [250, 500, 850], + standard_name='air_pressure', + var_name='plev', + units='hPa', + ) + lev_coord_hybrid_height = DimCoord( + [1.0, 0.5, 0.0], + standard_name='atmosphere_hybrid_height_coordinate', + var_name='lev', + units='m', + ) + lev_coord_hybrid_pressure = DimCoord( + [0.0, 0.5, 1.0], + standard_name='atmosphere_hybrid_sigma_pressure_coordinate', + var_name='lev', + units='1', + ) + ap_coord = AuxCoord( + [0.0, 0.0, 0.0], + var_name='ap', + units='Pa', + ) + b_coord = AuxCoord( + [0.0, 0.5, 1.0], + var_name='b', + units='1', + ) + ps_coord = AuxCoord( + np.full((2, 2, 2), 10), + var_name='ps', + units='Pa', + ) + orog_coord = AuxCoord( + np.full((2, 2), 10), + var_name='orog', + units='m', + ) + hybrid_height_factory = HybridHeightFactory( + delta=lev_coord_hybrid_height, + sigma=b_coord, + orography=orog_coord, + ) + hybrid_pressure_factory = HybridPressureFactory( + delta=ap_coord, + sigma=lev_coord_hybrid_pressure, + surface_air_pressure=ps_coord, + ) + lat_coord = DimCoord( + [0, 10], + standard_name='latitude', + var_name='lat', + units='degrees', + ) + lat_coord_rev = DimCoord( + [10, -10], + standard_name='latitude', + var_name='lat', + units='degrees', + ) + lat_coord_2d = AuxCoord( + [[10, -10]], + standard_name='latitude', + var_name='wrong_name', + units='degrees', + ) + lon_coord = DimCoord( + [-180, 0], + standard_name='longitude', + var_name='lon', + units='degrees', + ) + lon_coord_unstructured = AuxCoord( + [-180, 0], + bounds=[[-200, -180, -160], [-20, 0, 20]], + standard_name='longitude', + var_name='lon', + units='degrees', + ) + lon_coord_2d = AuxCoord( + [[370, 380]], + standard_name='longitude', + var_name='wrong_name', + units='degrees', + ) + height2m_coord = AuxCoord( + 2.0, + standard_name='height', + var_name='height', + units='m', + ) + + coord_spec_3d = [ + (time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + self.cube_3d = Cube( + da.arange(2 * 2 * 2, dtype=np.float32).reshape(2, 2, 2), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='tas', + units='celsius', + dim_coords_and_dims=coord_spec_3d, + aux_coords_and_dims=[(height2m_coord, ())], + attributes={}, + ) + + coord_spec_4d = [ + (time_coord, 0), + (plev_coord_rev, 1), + (lat_coord_rev, 2), + (lon_coord, 3), + ] + cube_4d = Cube( + da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='ta', + units='celsius', + dim_coords_and_dims=coord_spec_4d, + attributes={}, + ) + self.cubes_4d = CubeList([cube_4d]) + + coord_spec_hybrid_height_4d = [ + (time_coord, 0), + (lev_coord_hybrid_height, 1), + (lat_coord_rev, 2), + (lon_coord, 3), + ] + aux_coord_spec_hybrid_height_4d = [ + (b_coord, 1), + (orog_coord, (2, 3)), + ] + cube_hybrid_height_4d = Cube( + da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='ta', + units='celsius', + dim_coords_and_dims=coord_spec_hybrid_height_4d, + aux_coords_and_dims=aux_coord_spec_hybrid_height_4d, + aux_factories=[hybrid_height_factory], + attributes={}, + ) + self.cubes_hybrid_height_4d = CubeList([cube_hybrid_height_4d]) + + coord_spec_hybrid_pressure_4d = [ + (time_coord, 0), + (lev_coord_hybrid_pressure, 1), + (lat_coord_rev, 2), + (lon_coord, 3), + ] + aux_coord_spec_hybrid_pressure_4d = [ + (ap_coord, 1), + (ps_coord, (0, 2, 3)), + ] + cube_hybrid_pressure_4d = Cube( + da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='ta', + units='celsius', + dim_coords_and_dims=coord_spec_hybrid_pressure_4d, + aux_coords_and_dims=aux_coord_spec_hybrid_pressure_4d, + aux_factories=[hybrid_pressure_factory], + attributes={}, + ) + self.cubes_hybrid_pressure_4d = CubeList([cube_hybrid_pressure_4d]) + + coord_spec_unstrucutred = [ + (height2m_coord, ()), + (lat_coord_rev, 1), + (lon_coord_unstructured, 1), + ] + cube_unstructured = Cube( + da.zeros((2, 2)), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='tas', + units='celsius', + dim_coords_and_dims=[(time_coord, 0)], + aux_coords_and_dims=coord_spec_unstrucutred, + attributes={}, + ) + self.cubes_unstructured = CubeList([cube_unstructured]) + + coord_spec_2d = [ + (height2m_coord, ()), + (lat_coord_2d, (1, 2)), + (lon_coord_2d, (1, 2)), + ] + cube_2d_latlon = Cube( + da.zeros((2, 1, 2)), + standard_name='air_pressure', + long_name='Air Pressure', + var_name='tas', + units='celsius', + dim_coords_and_dims=[(time_coord, 0)], + aux_coords_and_dims=coord_spec_2d, + attributes={}, + ) + self.cubes_2d_latlon = CubeList([cube_2d_latlon]) + + def assert_time_metadata(self, cube): + """Assert time metadata is correct.""" + assert cube.coord('time').standard_name == 'time' + assert cube.coord('time').var_name == 'time' + assert cube.coord('time').units == Unit( + 'days since 1850-01-01', calendar='365_day' + ) + assert cube.coord('time').attributes == {'test': 1} + + def assert_time_data(self, cube, time_has_bounds=True): + """Assert time data is correct.""" + np.testing.assert_allclose(cube.coord('time').points, [380, 410]) + if time_has_bounds: + np.testing.assert_allclose( + cube.coord('time').bounds, [[365, 396], [396, 424]], + ) + else: + assert cube.coord('time').bounds is None + + def assert_plev_metadata(self, cube): + """Assert plev metadata is correct.""" + assert cube.coord('air_pressure').standard_name == 'air_pressure' + assert cube.coord('air_pressure').var_name == 'plev' + assert cube.coord('air_pressure').units == 'Pa' + assert cube.coord('air_pressure').attributes == {} + + def assert_lat_metadata(self, cube): + """Assert lat metadata is correct.""" + assert cube.coord('latitude').standard_name == 'latitude' + assert cube.coord('latitude').var_name == 'lat' + assert str(cube.coord('latitude').units) == 'degrees_north' + assert cube.coord('latitude').attributes == {} + + def assert_lon_metadata(self, cube): + """Assert lon metadata is correct.""" + assert cube.coord('longitude').standard_name == 'longitude' + assert cube.coord('longitude').var_name == 'lon' + assert str(cube.coord('longitude').units) == 'degrees_east' + assert cube.coord('longitude').attributes == {} + + def assert_ta_metadata(self, cube): + """Assert ta metadata is correct.""" + # Variable metadata + assert cube.standard_name == 'air_temperature' + assert cube.long_name == 'Air Temperature' + assert cube.var_name == 'ta' + assert cube.units == 'K' + assert cube.attributes == {} + + def assert_ta_data(self, cube, time_has_bounds=True): + """Assert ta data is correct.""" + assert cube.has_lazy_data() + np.testing.assert_allclose( + cube.data, + [[[[284.15, 283.15], + [282.15, 281.15]], + [[280.15, 279.15], + [278.15, 277.15]], + [[276.15, 275.15], + [274.15, 273.15]]], + [[[296.15, 295.15], + [294.15, 293.15]], + [[292.15, 291.15], + [290.15, 289.15]], + [[288.15, 287.15], + [286.15, 285.15]]]], + ) + + # Time + self.assert_time_data(cube, time_has_bounds=time_has_bounds) + + # Air pressure + np.testing.assert_allclose( + cube.coord('air_pressure').points, + [85000.0, 50000.0, 25000.0], + atol=1e-8, + ) + assert cube.coord('air_pressure').bounds is None + + # Latitude + np.testing.assert_allclose( + cube.coord('latitude').points, [-10.0, 10.0] + ) + np.testing.assert_allclose( + cube.coord('latitude').bounds, [[-20.0, 0.0], [0.0, 20.0]] + ) + + # Longitude + np.testing.assert_allclose( + cube.coord('longitude').points, [0.0, 180.0] + ) + np.testing.assert_allclose( + cube.coord('longitude').bounds, [[-90.0, 90.0], [90.0, 270.0]] + ) + + def assert_tas_metadata(self, cube): + """Assert tas metadata is correct.""" + assert cube.standard_name == 'air_temperature' + assert cube.long_name == 'Near-Surface Air Temperature' + assert cube.var_name == 'tas' + assert cube.units == 'K' + assert cube.attributes == {} + + # Height 2m coordinate + assert cube.coord('height').standard_name == 'height' + assert cube.coord('height').var_name == 'height' + assert cube.coord('height').units == 'm' + assert cube.coord('height').attributes == {} + np.testing.assert_allclose(cube.coord('height').points, 2.0) + assert cube.coord('height').bounds is None + + def test_fix_metadata_amon_ta(self): + """Test ``fix_metadata``.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + fixed_cubes = fix_metadata( + self.cubes_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_plev_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + self.assert_ta_data(fixed_cube) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 9 + + def test_fix_metadata_amon_ta_wrong_lat_units(self): + """Test ``fix_metadata``.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + # Change units of latitude + self.cubes_4d[0].coord('latitude').units = 'K' + + fixed_cubes = fix_metadata( + self.cubes_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_plev_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + self.assert_ta_data(fixed_cube) + + # CMOR check will fail because of wrong latitude units + assert fixed_cube.coord('latitude').units == 'K' + with pytest.raises(CMORCheckError): + cmor_check_metadata(fixed_cube, project, mip, short_name) + + print(self.mock_debug.mock_calls) + print(self.mock_warning.mock_calls) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 9 + + def test_fix_metadata_cfmon_ta_hybrid_height(self): + """Test ``fix_metadata`` with hybrid height coordinate.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'CFmon' + + fixed_cubes = fix_metadata( + self.cubes_hybrid_height_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + hybrid_coord = fixed_cube.coord('atmosphere_hybrid_height_coordinate') + assert hybrid_coord.var_name == 'lev' + assert hybrid_coord.long_name is None + assert hybrid_coord.units == 'm' + np.testing.assert_allclose(hybrid_coord.points, [0.0, 0.5, 1.0]) + assert fixed_cube.coords('altitude') + assert fixed_cube.coord_dims('altitude') == (1, 2, 3) + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 4 + assert self.mock_warning.call_count == 9 + + def test_fix_metadata_cfmon_ta_hybrid_pressure(self): + """Test ``fix_metadata`` with hybrid pressure coordinate.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'CFmon' + + fixed_cubes = fix_metadata( + self.cubes_hybrid_pressure_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + hybrid_coord = fixed_cube.coord( + 'atmosphere_hybrid_sigma_pressure_coordinate' + ) + assert hybrid_coord.var_name == 'lev' + assert hybrid_coord.long_name is None + assert hybrid_coord.units == '1' + np.testing.assert_allclose(hybrid_coord.points, [1.0, 0.5, 0.0]) + assert fixed_cube.coords('air_pressure') + assert fixed_cube.coord_dims('air_pressure') == (0, 1, 2, 3) + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 4 + assert self.mock_warning.call_count == 9 + + def test_fix_metadata_cfmon_ta_alternative(self): + """Test ``fix_metadata`` with alternative generic level coordinate.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'CFmon' + + fixed_cubes = fix_metadata( + self.cubes_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_plev_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + self.assert_ta_data(fixed_cube) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 4 + assert self.mock_warning.call_count == 9 + + def test_fix_metadata_cfmon_ta_no_alternative(self, mocker): + """Test ``fix_metadata`` with no alternative coordinate.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'CFmon' + + # Remove alternative coordinate + self.cubes_4d[0].remove_coord('air_pressure') + + fixed_cubes = fix_metadata( + self.cubes_4d, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_time_data(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + # CMOR check will fail because of missing alevel coordinate + assert not fixed_cube.coords('air_pressure') + with pytest.raises(CMORCheckError): + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 2 + assert self.mock_warning.call_count == 8 + + def test_fix_metadata_e1hr_ta(self): + """Test ``fix_metadata`` with plev3.""" + short_name = 'ta' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'E1hr' + + # Slightly adapt plev to test fixing of requested levels + self.cubes_4d[0].coord('air_pressure').points = [ + 250.0 + 9e-8, 500.0 + 9e-8, 850.0 + 9e-8 + ] + + fixed_cubes = fix_metadata( + self.cubes_4d, + short_name, + project, + dataset, + mip, + frequency='mon', + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_ta_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_plev_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + self.assert_ta_data(fixed_cube, time_has_bounds=False) + + cmor_check_metadata( + fixed_cube, project, mip, short_name, frequency='mon' + ) + + assert self.mock_debug.call_count == 4 + assert self.mock_warning.call_count == 8 + + def test_fix_metadata_amon_tas_unstructured(self): + """Test ``fix_metadata`` with unstructured grid.""" + short_name = 'tas' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + fixed_cubes = fix_metadata( + self.cubes_unstructured, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_tas_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + # Latitude + np.testing.assert_allclose( + fixed_cube.coord('latitude').points, [10.0, -10.0] + ) + assert fixed_cube.coord('latitude').bounds is None + + # Longitude + np.testing.assert_allclose( + fixed_cube.coord('longitude').points, [180.0, 0.0] + ) + np.testing.assert_allclose( + fixed_cube.coord('longitude').bounds, + [[160.0, 180.0, 200.0], [340.0, 0.0, 20.0]], + ) + + # Variable data + assert fixed_cube.has_lazy_data() + np.testing.assert_allclose( + fixed_cube.data, [[273.15, 273.15], [273.15, 273.15]] + ) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 2 + assert self.mock_warning.call_count == 6 + + def test_fix_metadata_amon_tas_2d_latlon(self): + """Test ``fix_metadata`` with 2D latitude/longitude.""" + short_name = 'tas' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + fixed_cubes = fix_metadata( + self.cubes_2d_latlon, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_tas_metadata(fixed_cube) + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + # Latitude + np.testing.assert_allclose( + fixed_cube.coord('latitude').points, [[10.0, -10.0]] + ) + assert fixed_cube.coord('latitude').bounds is None + + # Longitude + np.testing.assert_allclose( + fixed_cube.coord('longitude').points, [[10.0, 20.0]] + ) + assert fixed_cube.coord('longitude').bounds is None + + # Variable data + assert fixed_cube.has_lazy_data() + np.testing.assert_allclose( + fixed_cube.data, [[[273.15, 273.15]], [[273.15, 273.15]]] + ) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 8 + + def test_fix_metadata_amon_tas_invalid_time_units(self): + """Test ``fix_metadata`` with invalid time units.""" + short_name = 'tas' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + self.cubes_2d_latlon[0].remove_coord('time') + aux_time_coord = AuxCoord( + [1, 2], + standard_name='time', + var_name='time', + units='kg', + ) + self.cubes_2d_latlon[0].add_aux_coord(aux_time_coord, 0) + + fixed_cubes = fix_metadata( + self.cubes_2d_latlon, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + assert fixed_cube.coord('time').units == 'kg' + + # CMOR checks fail because calendar is not defined + with pytest.raises(ValueError): + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 7 + + def test_fix_metadata_amon_tas_invalid_time_attrs(self): + """Test ``fix_metadata`` with invalid time attributes.""" + short_name = 'tas' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + self.cubes_2d_latlon[0].attributes = { + 'parent_time_units': 'this is certainly not a unit', + 'branch_time_in_parent': 'BRANCH TIME IN PARENT', + 'branch_time_in_child': 'BRANCH TIME IN CHILD', + } + + fixed_cubes = fix_metadata( + self.cubes_2d_latlon, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + assert fixed_cube.attributes == { + 'parent_time_units': 'this is certainly not a unit', + 'branch_time_in_parent': 'BRANCH TIME IN PARENT', + 'branch_time_in_child': 'BRANCH TIME IN CHILD', + } + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 8 + + def test_fix_metadata_oimon_ssi(self): + """Test ``fix_metadata`` with psu units.""" + short_name = 'ssi' + project = 'CMIP5' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'OImon' + + self.cubes_2d_latlon[0].var_name = 'ssi' + self.cubes_2d_latlon[0].attributes = { + 'invalid_units': 'psu', + 'parent_time_units': 'no parent', + } + + # Also test 2D longitude that already has bounds + self.cubes_2d_latlon[0].coord('latitude').var_name = 'lat' + self.cubes_2d_latlon[0].coord('longitude').var_name = 'lon' + self.cubes_2d_latlon[0].coord('longitude').bounds = [ + [[365.0, 375.0], [375.0, 400.0]] + ] + + fixed_cubes = fix_metadata( + self.cubes_2d_latlon, + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + # Variable metadata + assert fixed_cube.standard_name == 'sea_ice_salinity' + assert fixed_cube.long_name == 'Sea Ice Salinity' + assert fixed_cube.var_name == 'ssi' + assert fixed_cube.units == '1' + assert fixed_cube.attributes == {'parent_time_units': 'no parent'} + + # Coordinates + self.assert_time_metadata(fixed_cube) + self.assert_lat_metadata(fixed_cube) + self.assert_lon_metadata(fixed_cube) + + # Latitude + np.testing.assert_allclose( + fixed_cube.coord('latitude').points, [[10.0, -10.0]] + ) + assert fixed_cube.coord('latitude').bounds is None + + # Longitude + np.testing.assert_allclose( + fixed_cube.coord('longitude').points, [[10.0, 20.0]] + ) + np.testing.assert_allclose( + fixed_cube.coord('longitude').bounds, + [[[5.0, 15.0], [15.0, 40.0]]], + ) + + # Variable data + assert fixed_cube.has_lazy_data() + np.testing.assert_allclose( + fixed_cube.data, [[[0.0, 0.0]], [[0.0, 0.0]]], + ) + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 2 + assert self.mock_warning.call_count == 6 + + def test_fix_data_amon_tas(self): + """Test ``fix_data``.""" + short_name = 'tas' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'Amon' + + fixed_cube = fix_data( + self.cube_3d, + short_name, + project, + dataset, + mip, + ) + + assert fixed_cube.has_lazy_data() + + cmor_check_data(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 0 + assert self.mock_warning.call_count == 0 + + def test_deprecate_check_level_fix_metadata(self): + """Test deprecation of check level in ``fix_metadata``.""" + with pytest.warns(ESMValCoreDeprecationWarning): + fix_metadata( + self.cubes_4d, + 'ta', + 'CMIP6', + 'MODEL', + 'Amon', + check_level=CheckLevels.RELAXED, + ) + + def test_deprecate_check_level_fix_data(self): + """Test deprecation of check level in ``fix_data``.""" + with pytest.warns(ESMValCoreDeprecationWarning): + fix_metadata( + self.cubes_4d, + 'ta', + 'CMIP6', + 'MODEL', + 'Amon', + check_level=CheckLevels.RELAXED, + ) diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 7b25b8d719..8b8d63e57c 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -3,6 +3,8 @@ import os import unittest +import pytest + import esmvalcore.cmor from esmvalcore.cmor.table import ( CMIP3Info, @@ -10,6 +12,7 @@ CMIP6Info, CustomInfo, _update_cmor_facets, + get_var_info, ) @@ -485,3 +488,41 @@ def test_get_variable_ch4s(self): self.assertEqual(var.long_name, 'Atmosphere CH4 surface') self.assertEqual(var.units, '1e-09') + + +@pytest.mark.parametrize( + 'project,mip,short_name,frequency', + [ + ('CMIP5', 'Amon', 'tas', 'mon'), + ('CMIP5', 'day', 'tas', 'day'), + ('CMIP6', 'Amon', 'tas', 'mon'), + ('CMIP6', 'day', 'tas', 'day'), + ('CORDEX', '3hr', 'tas', '3hr'), + ] +) +def test_get_var_info(project, mip, short_name, frequency): + """Test ``get_var_info``.""" + var_info = get_var_info(project, mip, short_name) + + assert var_info.short_name == short_name + assert var_info.frequency == frequency + + +@pytest.mark.parametrize( + 'mip,short_name', + [ + ('INVALID_MIP', 'tas'), + ('Amon', 'INVALID_VAR'), + ] +) +def test_get_var_info_invalid_mip_short_name(mip, short_name): + """Test ``get_var_info``.""" + var_info = get_var_info('CMIP6', mip, short_name) + + assert var_info is None + + +def test_get_var_info_invalid_project(): + """Test ``get_var_info``.""" + with pytest.raises(KeyError): + get_var_info('INVALID_PROJECT', 'Amon', 'tas') diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index f3135558a2..957136fd42 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -1,5 +1,6 @@ """Unit tests for the CMORCheck class.""" +import logging import unittest from copy import deepcopy @@ -10,6 +11,7 @@ import iris.cube import iris.util import numpy as np +import pytest from cf_units import Unit from esmvalcore.cmor.check import ( @@ -18,6 +20,9 @@ CMORCheckError, _get_cmor_checker, ) +from esmvalcore.exceptions import ESMValCoreDeprecationWarning + +logger = logging.getLogger(__name__) class VariableInfoMock: @@ -142,33 +147,39 @@ def test_check(self): """Test checks succeeds for a good cube.""" self._check_cube() - def _check_cube(self, automatic_fixes=False, frequency=None, + def _check_cube(self, frequency=None, check_level=CheckLevels.DEFAULT): - """Apply checks and optionally automatic fixes to self.cube.""" + """Apply checks to self.cube.""" def checker(cube): return CMORCheck( cube, self.var_info, - automatic_fixes=automatic_fixes, frequency=frequency, check_level=check_level) self.cube = checker(self.cube).check_metadata() self.cube = checker(self.cube).check_data() - def _check_cube_metadata(self, automatic_fixes=False, frequency=None, + def _check_cube_metadata(self, frequency=None, check_level=CheckLevels.DEFAULT): - """Apply checks and optionally automatic fixes to self.cube.""" + """Apply checks to self.cube.""" def checker(cube): return CMORCheck( cube, self.var_info, - automatic_fixes=automatic_fixes, frequency=frequency, check_level=check_level) self.cube = checker(self.cube).check_metadata() + def test_check_with_custom_logger(self): + """Test checks with custom logger.""" + def checker(cube): + return CMORCheck(cube, self.var_info) + + self.cube = checker(self.cube).check_metadata(logger=logger) + self.cube = checker(self.cube).check_data(logger=logger) + def test_check_with_month_number(self): """Test checks succeeds for a good cube with month number.""" iris.coord_categorisation.add_month_number(self.cube, 'time') @@ -189,13 +200,6 @@ def test_check_with_year(self): iris.coord_categorisation.add_year(self.cube, 'time') self._check_cube() - def test_check_with_time_attributes(self): - """Test checks succeeds for a good cube with year.""" - self.cube.coord('time').attributes['time_origin'] = "cow" - assert self.cube.coord('time').attributes['time_origin'] == "cow" - self._check_cube() - assert 'time_origin' not in self.cube.coord('time').attributes - def test_check_no_multiple_coords_same_stdname(self): """Test checks fails if two coords have the same standard_name.""" self.cube.add_aux_coord( @@ -209,43 +213,20 @@ def test_check_no_multiple_coords_same_stdname(self): ) self._check_fails_in_metadata() - def test_check_bad_standard_name_auto_fix(self): - """Test check pass for a bad standard_name with automatic fixes.""" - self.cube.standard_name = 'wind_speed' - self._check_cube(automatic_fixes=True) - self._check_cube() - def test_check_bad_standard_name(self): """Test check fails for a bad short_name.""" self.cube.standard_name = 'wind_speed' - self._check_fails_in_metadata(automatic_fixes=False) - - def test_check_bad_long_name_auto_fix(self): - """Test check pass for a bad standard_name with automatic fixes.""" - self.cube.long_name = 'bad_name' - self._check_cube(automatic_fixes=True) - self._check_cube() - - def test_check_bad_long_name_auto_fix_report_warning(self): - """Test check pass for a bad standard_name with automatic fixes.""" - self.cube.long_name = 'bad_name' - self._check_warnings_on_metadata(automatic_fixes=True) + self._check_fails_in_metadata() def test_check_bad_long_name(self): """Test check fails for a bad short_name.""" self.cube.long_name = 'bad_name' self._check_fails_in_metadata() - def test_check_with_unit_conversion(self): - """Test check succeeds for a good cube requiring unit conversion.""" + def test_check_bad_units(self): + """Test check fails for bad units.""" self.cube.units = 'days' - self._check_cube() - - def test_check_with_psu_units(self): - """Test check succeeds for a good cube with psu units.""" - self.var_info.units = 'psu' - self.cube = self.get_cube(self.var_info) - self._check_cube(automatic_fixes=True) + self._check_fails_in_metadata() def test_check_with_positive(self): """Check variable with positive attribute.""" @@ -288,25 +269,6 @@ def test_rank_unstructured_grid(self): self.cube = self._get_unstructed_grid_cube() self._check_cube() - def test_unstructured_grid_automatic_fixes(self): - """Check succeeds even if two required coordinates share dimensions.""" - self.cube = self._get_unstructed_grid_cube(correct_lon=False, - n_bounds=3) - - lon_coord = self.cube.coord('longitude') - np.testing.assert_allclose(lon_coord.points.min(), -180.0) - np.testing.assert_allclose(lon_coord.points.max(), 162.0) - np.testing.assert_allclose(lon_coord.bounds.min(), -180.0) - np.testing.assert_allclose(lon_coord.bounds.max(), 171.0) - - self._check_cube(automatic_fixes=True) - - lon_coord = self.cube.coord('longitude') - np.testing.assert_allclose(lon_coord.points.min(), 0.0) - np.testing.assert_allclose(lon_coord.points.max(), 342.0) - np.testing.assert_allclose(lon_coord.bounds.min(), 0.0) - np.testing.assert_allclose(lon_coord.bounds.max(), 351.0) - def test_bad_generic_level(self): """Test check fails in metadata if generic level coord has wrong var_name.""" @@ -328,6 +290,14 @@ def test_valid_generic_level(self): checker.check_metadata() checker.check_data() + # TODO: remove in v2.12 + def test_valid_generic_level_automatic_fixes(self): + """Test valid generic level coordinate with automatic fixes.""" + self._setup_generic_level_var() + checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) + checker.check_metadata() + checker.check_data() + def test_invalid_generic_level(self): """Test invalid generic level coordinate.""" self._setup_generic_level_var() @@ -376,19 +346,19 @@ def test_check_bad_var_standard_name_strict_flag(self): """Test check fails for a bad variable standard_name with --cmor-check strict.""" self.cube.standard_name = 'wind_speed' - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_var_long_name_strict_flag(self): """Test check fails for a bad variable long_name with --cmor-check strict.""" self.cube.long_name = "Near-Surface Wind Speed" - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_var_units_strict_flag(self): """Test check fails for a bad variable units with --cmor-check strict.""" self.cube.units = "kg" - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_attributes_strict_flag(self): """Test check fails for a bad variable attribute with @@ -397,7 +367,7 @@ def test_check_bad_attributes_strict_flag(self): self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) self.cube.attributes['positive'] = "Wrong attribute" - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_rank_strict_flag(self): """Test check fails for a bad variable rank with @@ -405,32 +375,32 @@ def test_check_bad_rank_strict_flag(self): lat = iris.coords.AuxCoord.from_coord(self.cube.coord('latitude')) self.cube.remove_coord('latitude') self.cube.add_aux_coord(lat, self.cube.coord_dims('longitude')) - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_coord_var_name_strict_flag(self): """Test check fails for bad coord var_name with --cmor-check strict""" self.var_info.table_type = 'CMIP5' self.cube.coord('longitude').var_name = 'bad_name' - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_missing_lon_strict_flag(self): """Test check fails for missing longitude with --cmor-check strict""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('longitude') - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_missing_lat_strict_flag(self): """Test check fails for missing latitude with --cmor-check strict""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('latitude') - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_missing_time_strict_flag(self): """Test check fails for missing time with --cmor-check strict""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('time') - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_missing_coord_strict_flag(self): """Test check fails for missing coord other than lat and lon @@ -438,28 +408,25 @@ def test_check_missing_coord_strict_flag(self): self.var_info.coordinates.update( {'height2m': CoordinateInfoMock('height2m')} ) - self._check_fails_in_metadata(automatic_fixes=False) + self._check_fails_in_metadata() def test_check_bad_var_standard_name_relaxed_flag(self): """Test check reports warning for a bad variable standard_name with --cmor-check relaxed.""" self.cube.standard_name = 'wind_speed' - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_long_name_relaxed_flag(self): """Test check reports warning for a bad variable long_name with --cmor-check relaxed.""" self.cube.long_name = "Near-Surface Wind Speed" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_units_relaxed_flag(self): """Test check reports warning for a bad variable units with --cmor-check relaxed.""" self.cube.units = "kg" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_attributes_relaxed_flag(self): """Test check report warnings for a bad variable attribute with @@ -468,8 +435,7 @@ def test_check_bad_attributes_relaxed_flag(self): self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) self.cube.attributes['positive'] = "Wrong attribute" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_rank_relaxed_flag(self): """Test check report warnings for a bad variable rank with @@ -477,37 +443,32 @@ def test_check_bad_rank_relaxed_flag(self): lat = iris.coords.AuxCoord.from_coord(self.cube.coord('latitude')) self.cube.remove_coord('latitude') self.cube.add_aux_coord(lat, self.cube.coord_dims('longitude')) - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_coord_standard_name_relaxed_flag(self): """Test check reports warning for bad coord var_name with --cmor-check relaxed""" self.var_info.table_type = 'CMIP5' self.cube.coord('longitude').var_name = 'bad_name' - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_lon_relaxed_flag(self): """Test check fails for missing longitude with --cmor-check relaxed""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('longitude') - self._check_fails_in_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_lat_relaxed_flag(self): """Test check fails for missing latitude with --cmor-check relaxed""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('latitude') - self._check_fails_in_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_time_relaxed_flag(self): """Test check fails for missing latitude with --cmor-check relaxed""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('time') - self._check_fails_in_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_coord_relaxed_flag(self): """Test check reports warning for missing coord other than lat and lon @@ -515,29 +476,25 @@ def test_check_missing_coord_relaxed_flag(self): self.var_info.coordinates.update( {'height2m': CoordinateInfoMock('height2m')} ) - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.RELAXED) + self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_standard_name_none_flag(self): """Test check reports warning for a bad variable standard_name with --cmor-check ignore.""" self.cube.standard_name = 'wind_speed' - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_var_long_name_none_flag(self): """Test check reports warning for a bad variable long_name with --cmor-check ignore.""" self.cube.long_name = "Near-Surface Wind Speed" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_var_units_none_flag(self): """Test check reports warning for a bad variable unit with --cmor-check ignore.""" self.cube.units = "kg" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_attributes_none_flag(self): """Test check reports warning for a bad variable attribute with @@ -546,8 +503,7 @@ def test_check_bad_attributes_none_flag(self): self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) self.cube.attributes['positive'] = "Wrong attribute" - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_rank_none_flag(self): """Test check reports warning for a bad variable rank with @@ -555,40 +511,35 @@ def test_check_bad_rank_none_flag(self): lat = iris.coords.AuxCoord.from_coord(self.cube.coord('latitude')) self.cube.remove_coord('latitude') self.cube.add_aux_coord(lat, self.cube.coord_dims('longitude')) - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_coord_standard_name_none_flag(self): """Test check reports warning for bad coord var_name with --cmor-check ignore.""" self.var_info.table_type = 'CMIP5' self.cube.coord('longitude').var_name = 'bad_name' - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_lon_none_flag(self): """Test check reports warning for missing longitude with --cmor-check ignore""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('longitude') - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_lat_none_flag(self): """Test check reports warning for missing latitude with --cmor-check ignore""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('latitude') - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_time_none_flag(self): """Test check reports warning for missing time with --cmor-check ignore""" self.var_info.table_type = 'CMIP5' self.cube.remove_coord('time') - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_coord_none_flag(self): """Test check reports warning for missing coord other than lat, lon and @@ -596,8 +547,7 @@ def test_check_missing_coord_none_flag(self): self.var_info.coordinates.update( {'height2m': CoordinateInfoMock('height2m')} ) - self._check_warnings_on_metadata(automatic_fixes=False, - check_level=CheckLevels.IGNORE) + self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_lazy(self): """Test checker does not realise data or aux_coords.""" @@ -627,29 +577,28 @@ def test_check_lazy(self): self.assertTrue(self.cube.coord('longitude').has_lazy_points()) self.assertTrue(self.cube.has_lazy_data()) - def _check_fails_in_metadata(self, automatic_fixes=False, frequency=None, + def _check_fails_in_metadata(self, frequency=None, check_level=CheckLevels.DEFAULT): checker = CMORCheck( self.cube, self.var_info, - automatic_fixes=automatic_fixes, frequency=frequency, - check_level=check_level) + check_level=check_level, + ) with self.assertRaises(CMORCheckError): checker.check_metadata() - def _check_warnings_on_metadata(self, automatic_fixes=False, - check_level=CheckLevels.DEFAULT): + def _check_warnings_on_metadata(self, check_level=CheckLevels.DEFAULT): checker = CMORCheck( - self.cube, self.var_info, automatic_fixes=automatic_fixes, - check_level=check_level + self.cube, self.var_info, check_level=check_level ) checker.check_metadata() self.assertTrue(checker.has_warnings()) - def _check_debug_messages_on_metadata(self, automatic_fixes=False): + def _check_debug_messages_on_metadata(self): checker = CMORCheck( - self.cube, self.var_info, automatic_fixes=automatic_fixes) + self.cube, self.var_info, + ) checker.check_metadata() self.assertTrue(checker.has_debug_messages()) @@ -667,20 +616,6 @@ def test_non_requested(self): checker.check_metadata() self.assertTrue(checker.has_warnings()) - def test_almost_requested(self): - """Automatically fix tiny deviations from the requested values.""" - coord = self.cube.coord('air_pressure') - values = np.array(coord.points) - values[0] += 1.e-7 - values[-1] += 1.e-7 - self._update_coordinate_values(self.cube, coord, values) - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_metadata() - reference = np.array( - self.var_info.coordinates['air_pressure'].requested, dtype=float) - np.testing.assert_array_equal( - self.cube.coord('air_pressure').points, reference) - def test_requested_str_values(self): """ Warning if requested values are not present. @@ -728,65 +663,40 @@ def test_non_increasing(self): self._update_coordinate_values(self.cube, coord, values) self._check_fails_in_metadata() - def test_non_increasing_fix(self): - """Check automatic fix for direction.""" - coord = self.cube.coord('latitude') - values = np.linspace( - coord.points[-1], - coord.points[0], - len(coord.points) - ) - self._update_coordinate_values(self.cube, coord, values) - self._check_cube(automatic_fixes=True) - self._check_cube() - # test bounds are contiguous - bounds = self.cube.coord('latitude').bounds - right_bounds = bounds[:-2, 1] - left_bounds = bounds[1:-1, 0] - self.assertTrue(np.all(left_bounds == right_bounds)) - def test_non_decreasing(self): """Fail in metadata if decreasing coordinate is increasing.""" self.var_info.coordinates['lat'].stored_direction = 'decreasing' self._check_fails_in_metadata() - def test_non_decreasing_fix(self): - """Check automatic fix for non decreasing coordinate.""" - self.cube.data[0, 0, 0, 0, 0] = 70 + # TODO: remove in v2.12 + def test_non_decreasing_automatic_fix_metadata(self): + """Automatic fix for decreasing coordinate.""" self.var_info.coordinates['lat'].stored_direction = 'decreasing' - self._check_cube(automatic_fixes=True) - self._check_cube() - index = [0, 0, 0, 0, 0] - index[self.cube.coord_dims('latitude')[0]] = -1 - self.assertEqual(self.cube.data.item(tuple(index)), 70) - self.assertEqual(self.cube.data[0, 0, 0, 0, 0], 50) - cube_points = self.cube.coord('latitude').points - reference = np.linspace(90, -90, 20, endpoint=True) - for index in range(20): - np.testing.assert_allclose(cube_points[index], reference[index]) - # test bounds are contiguous - bounds = self.cube.coord('latitude').bounds - right_bounds = bounds[:-2, 1] - left_bounds = bounds[1:-1, 0] - self.assertTrue(np.all(left_bounds == right_bounds)) + checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) + checker.check_metadata() - def test_not_bounds(self): - """Warning if bounds are not available.""" - self.cube.coord('longitude').bounds = None - self._check_warnings_on_metadata(automatic_fixes=False) - self.assertFalse(self.cube.coord('longitude').has_bounds()) + # TODO: remove in v2.12 + def test_non_decreasing_automatic_fix_data(self): + """Automatic fix for decreasing coordinate.""" + self.var_info.coordinates['lat'].stored_direction = 'decreasing' + checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) + checker.check_data() - def test_guess_bounds_with_fixes(self): - """Warning if bounds added with automatic fixes.""" - self.cube.coord('longitude').bounds = None - self._check_warnings_on_metadata(automatic_fixes=True) - self.assertTrue(self.cube.coord('longitude').has_bounds()) + def test_lat_non_monotonic(self): + """Test fail for non monotonic latitude.""" + lat = self.cube.coord('latitude') + points = np.array(lat.points) + points[-1] = points[0] + dims = self.cube.coord_dims(lat) + self.cube.remove_coord(lat) + lat = iris.coords.AuxCoord.from_coord(lat) + self.cube.add_aux_coord(lat.copy(points), dims) + self._check_fails_in_metadata() - def test_not_guess_bounds_with_fixes(self): - """Warning if bounds added with automatic fixes.""" + def test_not_bounds(self): + """Warning if bounds are not available.""" self.cube.coord('longitude').bounds = None - self.var_info.coordinates['lon'].must_have_bounds = "no" - self._check_cube(automatic_fixes=True) + self._check_warnings_on_metadata() self.assertFalse(self.cube.coord('longitude').has_bounds()) def test_not_correct_lons(self): @@ -794,34 +704,15 @@ def test_not_correct_lons(self): self.cube = self.cube.intersection(longitude=(-180., 180.)) self._check_fails_in_metadata() - def test_lons_automatic_fix(self): - """Test automatic fixes for bad longitudes.""" - self.cube = self.cube.intersection(longitude=(-180., 180.)) - self._check_cube(automatic_fixes=True) - - def test_lons_automatic_fix_with_bounds(self): - """Test automatic fixes for bad longitudes with added bounds.""" - self.cube.coord('longitude').bounds = None - self.cube = self.cube.intersection(longitude=(-180., 180.)) - self._check_cube(automatic_fixes=True) - self.assertTrue(self.cube.coord('longitude').points.min() >= 0.) - self.assertTrue(self.cube.coord('longitude').points.max() <= 360.) - self.assertTrue(self.cube.coord('longitude').has_bounds()) - - def test_high_lons_automatic_fix(self): - """Test automatic fixes for high longitudes.""" - self.cube = self.cube.intersection(longitude=(180., 520.)) - self._check_cube(automatic_fixes=True) - - def test_too_high_lons_automatic_fix_not_done(self): - """Test automatic fixes for bad longitudes.""" + def test_high_lons(self): + """Test bad longitudes.""" self.cube = self.cube.intersection(longitude=(720., 1080.)) - self._check_fails_in_metadata(automatic_fixes=True) + self._check_fails_in_metadata() - def test_too_low_lons_automatic_fix_not_done(self): - """Test automatic fixes for bad longitudes.""" + def test_low_lons(self): + """Test bad longitudes.""" self.cube = self.cube.intersection(longitude=(-720., -360.)) - self._check_fails_in_metadata(automatic_fixes=True) + self._check_fails_in_metadata() def test_not_valid_min(self): """Fail if coordinate values below valid_min.""" @@ -862,49 +753,16 @@ def test_bad_units(self): self.cube.coord('latitude').units = 'degrees_n' self._check_fails_in_metadata() - def test_units_automatic_fix(self): - """Test automatic fix for bad coordinate units.""" - self.cube.coord('latitude').units = 'degrees_n' - self._check_cube(automatic_fixes=True) - - def test_units_automatic_fix_failed(self): - """Test automatic fix fail for incompatible coordinate units.""" + def test_non_convertible_units(self): + """Test fail for incompatible coordinate units.""" self.cube.coord('latitude').units = 'degC' - self._check_fails_in_metadata(automatic_fixes=True) + self._check_fails_in_metadata() def test_bad_time(self): """Fail if time have bad units.""" self.cube.coord('time').units = 'days' self._check_fails_in_metadata() - def test_time_units_automatic_fix(self): - """Test automatic fix for time units.""" - self.cube.coord('time').units = 'days since 1860-1-1 00:00:00' - self.cube.attributes['parent_time_units'] = 'days since ' \ - '1850-1-1 00:00:00' - self.cube.attributes['branch_time_in_parent'] = 0. - self.cube.attributes['branch_time_in_child'] = 0. - self._check_cube() - assert (self.cube.coord('time').units.origin == - 'days since 1850-1-1 00:00:00') - assert self.cube.attributes['parent_time_units'] == 'days since ' \ - '1850-1-1 00:00:00' - assert self.cube.attributes['branch_time_in_parent'] == 0. - assert self.cube.attributes['branch_time_in_child'] == 3652. - - def test_time_units_no_parent_exp(self): - """Test non-conversion of time units attributes for no parent exps.""" - self.cube.coord('time').units = 'days since 1860-1-1 00:00:00' - self.cube.attributes['parent_time_units'] = 'no parent' - self.cube.attributes['branch_time_in_parent'] = 0. - self.cube.attributes['branch_time_in_child'] = 0. - self._check_cube() - assert (self.cube.coord('time').units.origin == - 'days since 1850-1-1 00:00:00') - assert self.cube.attributes['parent_time_units'] == 'no parent' - assert self.cube.attributes['branch_time_in_parent'] == 0. - assert self.cube.attributes['branch_time_in_child'] == 0. - def test_wrong_parent_time_unit(self): """Test fail for wrong parent time units.""" self.cube.coord('time').units = 'days since 1860-1-1 00:00:00' @@ -916,13 +774,13 @@ def test_wrong_parent_time_unit(self): assert self.cube.attributes['branch_time_in_parent'] == 0. assert self.cube.attributes['branch_time_in_child'] == 0 - def test_time_automatic_fix_failed(self): - """Test automatic fix fail for incompatible time units.""" + def test_time_non_time_units(self): + """Test fail for incompatible time units.""" self.cube.coord('time').units = 'K' - self._check_fails_in_metadata(automatic_fixes=True) + self._check_fails_in_metadata() def test_time_non_monotonic(self): - """Test automatic fix fail for non monotonic times.""" + """Test fail for non monotonic times.""" time = self.cube.coord('time') points = np.array(time.points) points[-1] = points[0] @@ -930,61 +788,13 @@ def test_time_non_monotonic(self): self.cube.remove_coord(time) time = iris.coords.AuxCoord.from_coord(time) self.cube.add_aux_coord(time.copy(points), dims) - self._check_fails_in_metadata(automatic_fixes=True) + self._check_fails_in_metadata() def test_bad_standard_name(self): """Fail if coordinates have bad standard names at metadata step.""" self.cube.coord('time').standard_name = 'region' self._check_fails_in_metadata() - def test_bad_out_name_multidim_latitude(self): - """Warning if multidimensional lat has bad var_name at metadata""" - self.var_info.table_type = 'CMIP6' - self.cube.remove_coord('latitude') - self.cube.add_aux_coord( - iris.coords.AuxCoord( - np.reshape(np.linspace(-90, 90, num=20*20), (20, 20)), - var_name='bad_name', - standard_name='latitude', - units='degrees_north' - ), - (1, 2) - ) - self._check_debug_messages_on_metadata() - - def test_bad_out_name_multidim_longitude(self): - """Warning if multidimensional lon has bad var_name at metadata""" - self.var_info.table_type = 'CMIP6' - self.cube.remove_coord('longitude') - self.cube.add_aux_coord( - iris.coords.AuxCoord( - np.reshape(np.linspace(-90, 90, num=20*20), (20, 20)), - var_name='bad_name', - standard_name='longitude', - units='degrees_east' - ), - (1, 2) - ) - self._check_debug_messages_on_metadata(automatic_fixes=True) - - def test_bad_bounds_in_multidim_longitude(self): - """Warning if multidimensional lon has bad var_name at metadata""" - self.var_info.table_type = 'CMIP6' - self.cube.remove_coord('longitude') - lons = np.reshape(np.linspace(180, 540, num=20*20), (20, 20)) - bounds = np.stack([lons.copy() - 0.5, lons.copy() + 0.5], -1) - self.cube.add_aux_coord( - iris.coords.AuxCoord( - lons, - var_name='bad_name', - standard_name='longitude', - units='degrees_east', - bounds=bounds, - ), - (1, 2) - ) - self._check_debug_messages_on_metadata(automatic_fixes=True) - def test_bad_out_name_region_area_type(self): """Debug message if region/area_type AuxCoord has bad var_name at metadata.""" @@ -1086,49 +896,20 @@ def test_frequency_not_supported(self): """Fail at metadata if frequency is not supported.""" self._check_fails_in_metadata(frequency='wrong_freq') - def test_time_bounds(self): - """Test time bounds are guessed for all frequencies""" - freqs = ['hr', '3hr', '6hr', 'day', 'mon', 'yr', 'dec'] - guessed = [] - for freq in freqs: - cube = self.get_cube(self.var_info, frequency=freq) - cube.coord('time').bounds = None - self.cube = cube - self._check_cube(automatic_fixes=True, frequency=freq) - if cube.coord('time').bounds is not None: - guessed.append(True) - assert len(guessed) == len(freqs) - - def test_no_time_bounds(self): - """Test time bounds are not guessed for instantaneous data""" - self.var_info.coordinates['time'].must_have_bounds = 'no' - self.var_info.coordinates['time1'] = ( - self.var_info.coordinates.pop('time')) - self.cube.coord('time').bounds = None - self._check_cube(automatic_fixes=True) - guessed_bounds = self.cube.coord('time').bounds - assert guessed_bounds is None - def test_hr_mip_cordex(self): """Test hourly CORDEX tables are found.""" checker = _get_cmor_checker('CORDEX', '3hr', 'tas', '3hr') assert checker(self.cube)._cmor_var.short_name == 'tas' assert checker(self.cube)._cmor_var.frequency == '3hr' - def test_set_range_in_0_360(self): - checker = _get_cmor_checker('CMIP5', 'Amon', 'tas', 'mon') - arr_in = np.array([[-120.0, 0.0, 20.0], [-1.0, 360.0, 400.0]]) - arr_exp = np.array([[240.0, 0.0, 20.0], [359.0, 0.0, 40.0]]) - arr_out = checker(self.cube)._set_range_in_0_360(arr_in) - np.testing.assert_allclose(arr_out, arr_exp) - - def test_set_range_in_0_360_lazy(self): - checker = _get_cmor_checker('CMIP5', 'Amon', 'tas', 'mon') - arr_in = da.from_array([[-120.0, 0.0, 20.0], [-1.0, 360.0, 400.0]]) - arr_exp = da.from_array([[240.0, 0.0, 20.0], [359.0, 0.0, 40.0]]) - arr_out = checker(self.cube)._set_range_in_0_360(arr_in) - self.assertTrue(isinstance(arr_out, da.core.Array)) - np.testing.assert_allclose(arr_out.compute(), arr_exp.compute()) + def test_custom_variable(self): + checker = _get_cmor_checker('OBS', 'Amon', 'uajet', 'mon') + assert checker(self.cube)._cmor_var.short_name == 'uajet' + assert checker(self.cube)._cmor_var.long_name == ( + 'Jet position expressed as latitude of maximum meridional wind ' + 'speed' + ) + assert checker(self.cube)._cmor_var.units == 'degrees' def _check_fails_on_data(self): checker = CMORCheck(self.cube, self.var_info) @@ -1210,7 +991,7 @@ def get_cube(self, return cube - def _get_unstructed_grid_cube(self, correct_lon=True, n_bounds=2): + def _get_unstructed_grid_cube(self, n_bounds=2): """Get cube with unstructured grid.""" assert n_bounds in (2, 3), "Only 2 or 3 bounds per cell supported" @@ -1256,23 +1037,6 @@ def _get_unstructed_grid_cube(self, correct_lon=True, n_bounds=2): )) coord.bounds = np.swapaxes(new_bounds, 0, 1) - # Convert longitude to [-180, 180] to check automatic fixes if desired - if correct_lon: - return cube - new_lon = iris.coords.AuxCoord( - points=cube.coord('longitude').points, - bounds=cube.coord('longitude').bounds, - var_name='lon', - standard_name='longitude', - long_name='Longitude', - units='degrees_east', - ) - new_lon.points = np.where(new_lon.points < 180.0, new_lon.points, - new_lon.points - 360.0) - new_lon.bounds = np.where(new_lon.bounds < 180.0, new_lon.bounds, - new_lon.bounds - 360.0) - cube.remove_coord('longitude') - cube.add_aux_coord(new_lon, 1) return cube def _setup_generic_level_var(self): @@ -1450,5 +1214,17 @@ def _get_time_values(dim_spec): return np.arange(start, end, step=delta) +def test_get_cmor_checker_invalid_project_fail(): + """Test ``_get_cmor_checker`` with invalid project.""" + with pytest.raises(KeyError): + _get_cmor_checker('INVALID_PROJECT', 'mip', 'short_name', 'frequency') + + +def test_deprecate_automatic_fixes(): + """Test deprecation of automatic_fixes.""" + with pytest.warns(ESMValCoreDeprecationWarning): + CMORCheck('cube', 'var_info', 'frequency', automatic_fixes=True) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 86293e2aaa..d6a06b2c11 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -1,16 +1,17 @@ """Unit tests for :mod:`esmvalcore.cmor.fix`.""" from pathlib import Path -from unittest import TestCase from unittest.mock import Mock, patch, sentinel -from esmvalcore.cmor.check import CheckLevels +import pytest + from esmvalcore.cmor.fix import Fix, fix_data, fix_file, fix_metadata -class TestFixFile(TestCase): +class TestFixFile(): """Fix file tests.""" + @pytest.fixture(autouse=True) def setUp(self): """Prepare for testing.""" self.filename = 'filename' @@ -26,8 +27,10 @@ def setUp(self): 'dataset': 'model', 'mip': 'mip', 'short_name': 'short_name', + 'frequency': 'frequency', }, 'session': sentinel.session, + 'frequency': 'frequency', } def test_fix(self): @@ -42,9 +45,10 @@ def test_fix(self): mip='mip', output_dir=Path('output_dir'), session=sentinel.session, + frequency='frequency', ) - self.assertNotEqual(file_returned, self.filename) - self.assertEqual(file_returned, 'new_filename') + assert file_returned != self.filename + assert file_returned == 'new_filename' mock_get_fixes.assert_called_once_with( **self.expected_get_fixes_call ) @@ -61,16 +65,18 @@ def test_nofix(self): mip='mip', output_dir=Path('output_dir'), session=sentinel.session, + frequency='frequency', ) - self.assertEqual(file_returned, self.filename) + assert file_returned == self.filename mock_get_fixes.assert_called_once_with( **self.expected_get_fixes_call ) -class TestGetCube(TestCase): +class TestGetCube(): """Test get cube by var_name method.""" + @pytest.fixture(autouse=True) def setUp(self): """Prepare for testing.""" self.cube_1 = Mock() @@ -84,28 +90,27 @@ def setUp(self): def test_get_first_cube(self): """Test selecting first cube.""" - self.assertIs(self.cube_1, - self.fix.get_cube_from_list(self.cubes, "cube1")) + assert self.cube_1 is self.fix.get_cube_from_list(self.cubes, "cube1") def test_get_second_cube(self): """Test selecting second cube.""" - self.assertIs(self.cube_2, - self.fix.get_cube_from_list(self.cubes, "cube2")) + assert self.cube_2 is self.fix.get_cube_from_list(self.cubes, "cube2") def test_get_default_raises(self): """Check that the default raises (Fix is not a cube).""" - with self.assertRaises(Exception): + with pytest.raises(Exception): self.fix.get_cube_from_list(self.cubes) def test_get_default(self): """Check that the default return the cube (fix is a cube).""" self.cube_1.var_name = 'fix' - self.assertIs(self.cube_1, self.fix.get_cube_from_list(self.cubes)) + assert self.cube_1 is self.fix.get_cube_from_list(self.cubes) -class TestFixMetadata(TestCase): +class TestFixMetadata(): """Fix metadata tests.""" + @pytest.fixture(autouse=True) def setUp(self): """Prepare for testing.""" self.cube = self._create_mock_cube() @@ -125,9 +130,10 @@ def setUp(self): 'dataset': 'model', 'mip': 'mip', 'short_name': 'short_name', - 'frequency': None, + 'frequency': 'frequency', }, 'session': sentinel.session, + 'frequency': 'frequency', } @staticmethod @@ -150,9 +156,12 @@ def test_fix(self): project='project', dataset='model', mip='mip', + frequency='frequency', session=sentinel.session, )[0] - self.checker.assert_called_once_with(self.intermediate_cube) + self.checker.assert_called_once_with( + self.intermediate_cube + ) self.check_metadata.assert_called_once_with() assert cube_returned is not self.cube assert cube_returned is not self.intermediate_cube @@ -174,6 +183,7 @@ def test_nofix(self): project='project', dataset='model', mip='mip', + frequency='frequency', session=sentinel.session, )[0] self.checker.assert_called_once_with(self.cube) @@ -193,8 +203,7 @@ def test_select_var(self): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=self.checker): cube_returned = fix_metadata( - cubes=[self.cube, - self._create_mock_cube('extra')], + cubes=[self.cube, self._create_mock_cube('extra')], short_name='short_name', project='CMIP6', dataset='model', @@ -205,52 +214,25 @@ def test_select_var(self): assert cube_returned is self.cube def test_select_var_failed_if_bad_var_name(self): - """Check that the same cube is returned if no fix is available.""" - with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[]): - with self.assertRaises(ValueError): - fix_metadata( - cubes=[ - self._create_mock_cube('not_me'), - self._create_mock_cube('me_neither') - ], - short_name='short_name', - project='CMIP6', - dataset='model', - mip='mip', - ) - - def test_cmor_checker_called(self): - """Check that the cmor check is done.""" - checker = Mock() - checker.return_value = Mock() - with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[]): - with patch('esmvalcore.cmor.fix._get_cmor_checker', - return_value=checker) as get_mock: - fix_metadata( - cubes=[self.cube], - short_name='short_name', - project='CMIP6', - dataset='dataset', - mip='mip', - frequency='frequency', - ) - get_mock.assert_called_once_with( - automatic_fixes=True, - fail_on_error=False, - frequency='frequency', - mip='mip', - short_name='short_name', - table='CMIP6', - check_level=CheckLevels.DEFAULT,) - checker.assert_called_once_with(self.cube) - checker.return_value.check_metadata.assert_called_once_with() + """Check that an error is raised if short_names do not match.""" + msg = "More than one cube found for variable tas in CMIP6:model" + with pytest.raises(ValueError, match=msg): + fix_metadata( + cubes=[ + self._create_mock_cube('not_me'), + self._create_mock_cube('me_neither') + ], + short_name='tas', + project='CMIP6', + dataset='model', + mip='Amon', + ) -class TestFixData(TestCase): +class TestFixData(): """Fix data tests.""" + @pytest.fixture(autouse=True) def setUp(self): """Prepare for testing.""" self.cube = Mock() @@ -270,9 +252,10 @@ def setUp(self): 'dataset': 'model', 'mip': 'mip', 'short_name': 'short_name', - 'frequency': None, + 'frequency': 'frequency', }, 'session': sentinel.session, + 'frequency': 'frequency', } def test_fix(self): @@ -288,9 +271,12 @@ def test_fix(self): project='project', dataset='model', mip='mip', + frequency='frequency', session=sentinel.session, ) - self.checker.assert_called_once_with(self.intermediate_cube) + self.checker.assert_called_once_with( + self.intermediate_cube + ) self.check_data.assert_called_once_with() assert cube_returned is not self.cube assert cube_returned is not self.intermediate_cube @@ -312,6 +298,7 @@ def test_nofix(self): project='project', dataset='model', mip='mip', + frequency='frequency', session=sentinel.session, ) self.checker.assert_called_once_with(self.cube) @@ -322,25 +309,3 @@ def test_nofix(self): mock_get_fixes.assert_called_once_with( **self.expected_get_fixes_call ) - - def test_cmor_checker_called(self): - """Check that the cmor check is done.""" - checker = Mock() - checker.return_value = Mock() - with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', - return_value=[]): - with patch('esmvalcore.cmor.fix._get_cmor_checker', - return_value=checker) as get_mock: - fix_data(self.cube, 'short_name', 'CMIP6', 'model', 'mip', - 'frequency') - get_mock.assert_called_once_with( - table='CMIP6', - automatic_fixes=True, - check_level=CheckLevels.DEFAULT, - fail_on_error=False, - frequency='frequency', - mip='mip', - short_name='short_name', - ) - checker.assert_called_once_with(self.cube) - checker.return_value.check_data.assert_called_once_with() diff --git a/tests/unit/cmor/test_generic_fix.py b/tests/unit/cmor/test_generic_fix.py new file mode 100644 index 0000000000..54703b93f5 --- /dev/null +++ b/tests/unit/cmor/test_generic_fix.py @@ -0,0 +1,237 @@ +"""Unit tests for generic fixes.""" + +from unittest.mock import sentinel + +import numpy as np +import pytest +from iris.coords import AuxCoord +from iris.cube import Cube, CubeList + +from esmvalcore.cmor._fixes.fix import GenericFix, get_time_bounds +from esmvalcore.cmor.table import get_var_info + + +@pytest.fixture +def time_coord(): + """Time coordinate.""" + time_coord = AuxCoord( + [15, 350], + standard_name='time', + units='days since 1850-01-01' + ) + return time_coord + + +@pytest.fixture +def generic_fix(): + """Generic fix object.""" + vardef = get_var_info('CMIP6', 'CFmon', 'ta') + extra_facets = {'short_name': 'ta', 'project': 'CMIP6', 'dataset': 'MODEL'} + return GenericFix(vardef, extra_facets=extra_facets) + + +@pytest.mark.parametrize( + 'freq,expected_bounds', + [ + ('mon', [[0, 31], [334, 365]]), + ('mo', [[0, 31], [334, 365]]), + ('yr', [[0, 365], [0, 365]]), + ('dec', [[0, 3652], [0, 3652]]), + ('day', [[14.5, 15.5], [349.5, 350.5]]), + ('6hr', [[14.875, 15.125], [349.875, 350.125]]), + ('3hr', [[14.9375, 15.0625], [349.9375, 350.0625]]), + ('1hr', [[14.97916666, 15.020833333], [349.97916666, 350.020833333]]), + ] +) +def test_get_time_bounds(time_coord, freq, expected_bounds): + """Test ``get_time_bounds`.""" + bounds = get_time_bounds(time_coord, freq) + np.testing.assert_allclose(bounds, expected_bounds) + + +def test_get_time_bounds_invalid_freq_fail(time_coord): + """Test ``get_time_bounds`.""" + with pytest.raises(NotImplementedError): + get_time_bounds(time_coord, 'invalid_freq') + + +def test_generic_fix_empty_long_name(generic_fix, monkeypatch): + """Test ``GenericFix``.""" + # Artificially set long_name to empty string for test + monkeypatch.setattr(generic_fix.vardef, 'long_name', '') + + cube = generic_fix._fix_long_name(sentinel.cube) + + assert cube == sentinel.cube + + +def test_generic_fix_empty_units(generic_fix, monkeypatch): + """Test ``GenericFix``.""" + # Artificially set latitude units to empty string for test + coord_info = generic_fix.vardef.coordinates['latitude'] + monkeypatch.setattr(coord_info, 'units', '') + + ret = generic_fix._fix_coord_units( + sentinel.cube, coord_info, sentinel.cube_coord + ) + + assert ret is None + + +def test_generic_fix_no_generic_lev_coords(generic_fix, monkeypatch): + """Test ``GenericFix``.""" + # Artificially remove generic_lev_coords + monkeypatch.setattr( + generic_fix.vardef.coordinates['alevel'], 'generic_lev_coords', {} + ) + + cube = generic_fix._fix_alternative_generic_level_coords(sentinel.cube) + + assert cube == sentinel.cube + + +def test_requested_levels_2d_coord(generic_fix, mocker): + """Test ``GenericFix``.""" + cube_coord = AuxCoord([[0]], standard_name='latitude', units='rad') + cmor_coord = mocker.Mock(requested=True) + + ret = generic_fix._fix_requested_coord_values( + sentinel.cube, cmor_coord, cube_coord + ) + + assert ret is None + + +def test_requested_levels_invalid_arr(generic_fix, mocker): + """Test ``GenericFix``.""" + cube_coord = AuxCoord([0], standard_name='latitude', units='rad') + cmor_coord = mocker.Mock(requested=['a', 'b']) + + ret = generic_fix._fix_requested_coord_values( + sentinel.cube, cmor_coord, cube_coord + ) + + assert ret is None + + +def test_lon_no_fix_needed(generic_fix): + """Test ``GenericFix``.""" + cube_coord = AuxCoord( + [0.0, 180.0, 360.0], standard_name='longitude', units='rad' + ) + + ret = generic_fix._fix_longitude_0_360( + sentinel.cube, sentinel.cmor_coord, cube_coord + ) + + assert ret == (sentinel.cube, cube_coord) + + +def test_lon_too_low_to_fix(generic_fix): + """Test ``GenericFix``.""" + cube_coord = AuxCoord( + [-370.0, 0.0], standard_name='longitude', units='rad' + ) + + ret = generic_fix._fix_longitude_0_360( + sentinel.cube, sentinel.cmor_coord, cube_coord + ) + + assert ret == (sentinel.cube, cube_coord) + + +def test_lon_too_high_to_fix(generic_fix): + """Test ``GenericFix``.""" + cube_coord = AuxCoord([750.0, 0.0], standard_name='longitude', units='rad') + + ret = generic_fix._fix_longitude_0_360( + sentinel.cube, sentinel.cmor_coord, cube_coord + ) + + assert ret == (sentinel.cube, cube_coord) + + +def test_fix_direction_2d_coord(generic_fix): + """Test ``GenericFix``.""" + cube_coord = AuxCoord([[0]], standard_name='latitude', units='rad') + + ret = generic_fix._fix_coord_direction( + sentinel.cube, sentinel.cmor_coord, cube_coord + ) + + assert ret == (sentinel.cube, cube_coord) + + +def test_fix_direction_string_coord(generic_fix): + """Test ``GenericFix``.""" + cube_coord = AuxCoord(['a'], standard_name='latitude', units='rad') + + ret = generic_fix._fix_coord_direction( + sentinel.cube, sentinel.cmor_coord, cube_coord + ) + + assert ret == (sentinel.cube, cube_coord) + + +def test_fix_direction_no_stored_direction(generic_fix, mocker): + """Test ``GenericFix``.""" + cube = Cube(0) + cube_coord = AuxCoord([0, 1], standard_name='latitude', units='rad') + cmor_coord = mocker.Mock(stored_direction='') + + ret = generic_fix._fix_coord_direction(cube, cmor_coord, cube_coord) + + assert ret == (cube, cube_coord) + + +def test_fix_metadata_not_fail_with_empty_cube(generic_fix): + """Generic fixes should not fail with empty cubes.""" + fixed_cubes = generic_fix.fix_metadata([Cube(0)]) + + assert isinstance(fixed_cubes, CubeList) + assert len(fixed_cubes) == 1 + assert fixed_cubes[0] == Cube( + 0, standard_name='air_temperature', long_name='Air Temperature' + ) + + +@pytest.mark.parametrize( + 'extra_facets', [{}, {'project': 'P', 'dataset': 'D'}] +) +def test_fix_metadata_multiple_cubes_fail(extra_facets): + """Generic fixes should fail when multiple invalid cubes are given.""" + vardef = get_var_info('CMIP6', 'Amon', 'ta') + fix = GenericFix(vardef, extra_facets=extra_facets) + with pytest.raises(ValueError): + fix.fix_metadata([Cube(0), Cube(0)]) + + +def test_fix_metadata_no_extra_facets(): + """Generic fixes should not fail when no extra facets are given.""" + vardef = get_var_info('CMIP6', 'Amon', 'ta') + fix = GenericFix(vardef) + fixed_cubes = fix.fix_metadata([Cube(0)]) + + assert isinstance(fixed_cubes, CubeList) + assert len(fixed_cubes) == 1 + assert fixed_cubes[0] == Cube( + 0, standard_name='air_temperature', long_name='Air Temperature' + ) + + +def test_fix_data_not_fail_with_empty_cube(generic_fix): + """Generic fixes should not fail with empty cubes.""" + fixed_cube = generic_fix.fix_data(Cube(0)) + + assert isinstance(fixed_cube, Cube) + assert fixed_cube == Cube(0) + + +def test_fix_data_no_extra_facets(): + """Generic fixes should not fail when no extra facets are given.""" + vardef = get_var_info('CMIP6', 'Amon', 'ta') + fix = GenericFix(vardef) + fixed_cube = fix.fix_data(Cube(0)) + + assert isinstance(fixed_cube, Cube) + assert fixed_cube == Cube(0) diff --git a/tests/unit/cmor/test_utils.py b/tests/unit/cmor/test_utils.py new file mode 100644 index 0000000000..f373524a7d --- /dev/null +++ b/tests/unit/cmor/test_utils.py @@ -0,0 +1,60 @@ +"""Unit tests for :mod:`esmvalcore.cmor._utils`.""" + +import pytest +from iris.cube import Cube + +from esmvalcore.cmor._utils import _get_single_cube + + +@pytest.mark.parametrize( + 'cubes', [[Cube(0)], [Cube(0, var_name='x')], [Cube(0, var_name='y')]] +) +def test_get_single_cube_one_cube(cubes, caplog): + """Test ``_get_single_cube``.""" + single_cube = _get_single_cube(cubes, 'x') + assert single_cube == cubes[0] + assert not caplog.records + + +@pytest.mark.parametrize( + 'dataset_str,msg', [ + (None, "Found variable x, but"), + ('XYZ', "Found variable x in XYZ, but"), + ] +) +@pytest.mark.parametrize( + 'cubes', [ + [Cube(0), Cube(0, var_name='x')], + [Cube(0, var_name='x'), Cube(0)], + [Cube(0, var_name='x'), Cube(0, var_name='x')], + [Cube(0), Cube(0), Cube(0, var_name='x')], + ] +) +def test_get_single_cube_multiple_cubes(cubes, dataset_str, msg, caplog): + """Test ``_get_single_cube``.""" + single_cube = _get_single_cube(cubes, 'x', dataset_str=dataset_str) + assert single_cube == Cube(0, var_name='x') + assert len(caplog.records) == 1 + log = caplog.records[0] + assert log.levelname == 'WARNING' + assert msg in log.message + + +@pytest.mark.parametrize( + 'dataset_str,msg', [ + (None, "More than one cube found for variable x but"), + ('XYZ', "More than one cube found for variable x in XYZ but"), + ] +) +@pytest.mark.parametrize( + 'cubes', [ + [Cube(0), Cube(0)], + [Cube(0, var_name='y'), Cube(0)], + [Cube(0, var_name='y'), Cube(0, var_name='z')], + [Cube(0), Cube(0), Cube(0, var_name='z')], + ] +) +def test_get_single_cube_no_cubes_fail(cubes, dataset_str, msg): + """Test ``_get_single_cube``.""" + with pytest.raises(ValueError, match=msg): + _get_single_cube(cubes, 'x', dataset_str=dataset_str) diff --git a/tests/unit/test_cmor_api.py b/tests/unit/test_cmor_api.py index 781ce30150..ac61c5006f 100644 --- a/tests/unit/test_cmor_api.py +++ b/tests/unit/test_cmor_api.py @@ -1,4 +1,6 @@ # flake8: noqa +from unittest.mock import sentinel + import esmvalcore.cmor.check import esmvalcore.cmor.fix import esmvalcore.cmor.fixes @@ -11,3 +13,114 @@ cmor_check_data, cmor_check_metadata, ) + + +def test_cmor_check_metadata(mocker): + """Test ``cmor_check_metadata``""" + mock_get_cmor_checker = mocker.patch.object( + esmvalcore.cmor.check, '_get_cmor_checker', autospec=True + ) + ( + mock_get_cmor_checker.return_value.return_value.check_metadata. + return_value + ) = sentinel.checked_cube + + cube = cmor_check_metadata( + sentinel.cube, + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + + mock_get_cmor_checker.assert_called_once_with( + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + mock_get_cmor_checker.return_value.assert_called_once_with(sentinel.cube) + ( + mock_get_cmor_checker.return_value.return_value.check_metadata. + assert_called_once_with() + ) + assert cube == sentinel.checked_cube + + +def test_cmor_check_data(mocker): + """Test ``cmor_check_data``""" + mock_get_cmor_checker = mocker.patch.object( + esmvalcore.cmor.check, '_get_cmor_checker', autospec=True + ) + ( + mock_get_cmor_checker.return_value.return_value.check_data. + return_value + ) = sentinel.checked_cube + + cube = cmor_check_data( + sentinel.cube, + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + + mock_get_cmor_checker.assert_called_once_with( + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + mock_get_cmor_checker.return_value.assert_called_once_with(sentinel.cube) + ( + mock_get_cmor_checker.return_value.return_value.check_data. + assert_called_once_with() + ) + assert cube == sentinel.checked_cube + + +def test_cmor_check(mocker): + """Test ``cmor_check``""" + mock_cmor_check_metadata = mocker.patch.object( + esmvalcore.cmor.check, + 'cmor_check_metadata', + autospec=True, + return_value=sentinel.cube_after_check_metadata, + ) + mock_cmor_check_data = mocker.patch.object( + esmvalcore.cmor.check, + 'cmor_check_data', + autospec=True, + return_value=sentinel.cube_after_check_data, + ) + + cube = cmor_check( + sentinel.cube, + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + sentinel.check_level, + ) + + mock_cmor_check_metadata.assert_called_once_with( + sentinel.cube, + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + mock_cmor_check_data.assert_called_once_with( + sentinel.cube_after_check_metadata, + sentinel.cmor_table, + sentinel.mip, + sentinel.short_name, + sentinel.frequency, + check_level=sentinel.check_level, + ) + assert cube == sentinel.cube_after_check_data From be6009e698ab87b92285fd3eb9ed77b16df97d2d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 14:19:41 +0200 Subject: [PATCH 08/72] Rechunk between preprocessor steps (#2205) --- esmvalcore/preprocessor/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 736ed02156..0cabc5d481 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -371,6 +371,10 @@ def preprocess( function = globals()[step] itype = _get_itype(step) + for item in items: + if isinstance(item, Cube) and item.has_lazy_data(): + item.data = item.core_data().rechunk() + result = [] if itype.endswith('s'): result.append(_run_preproc_function(function, items, settings, From c62ddc6b72c126732edeb6cdf8ed38acd1209728 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 14:23:35 +0200 Subject: [PATCH 09/72] Remove deprecated callback argument from preprocessor ``load`` function (#2207) --- esmvalcore/_recipe/recipe.py | 3 --- esmvalcore/dataset.py | 11 +++-------- esmvalcore/preprocessor/__init__.py | 5 +---- esmvalcore/preprocessor/_io.py | 19 ++----------------- .../integration/preprocessor/_io/test_load.py | 8 ++++---- .../preprocessor/test_preprocessing_task.py | 6 +++--- tests/integration/recipe/test_recipe.py | 7 ------- tests/unit/recipe/test_recipe.py | 1 - tests/unit/test_dataset.py | 1 - 9 files changed, 13 insertions(+), 48 deletions(-) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index ed858e0c39..6f003e17e0 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -213,9 +213,6 @@ def _get_default_settings(dataset): settings = {} - # Configure (deprecated, remove for v2.10.0) load callback - settings['load'] = {'callback': 'default'} - if _derive_needed(dataset): settings['derive'] = { 'short_name': facets['short_name'], diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 35c8588404..849a9a579e 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -677,19 +677,15 @@ def load(self) -> Cube: iris.cube.Cube An :mod:`iris` cube with the data corresponding the the dataset. """ - return self._load_with_callback(callback='default') - - def _load_with_callback(self, callback): - # TODO: Remove the callback argument for v2.10.0. input_files = list(self.files) for supplementary_dataset in self.supplementaries: input_files.extend(supplementary_dataset.files) esgf.download(input_files, self.session['download_dir']) - cube = self._load(callback) + cube = self._load() supplementary_cubes = [] for supplementary_dataset in self.supplementaries: - supplementary_cube = supplementary_dataset._load(callback) + supplementary_cube = supplementary_dataset._load() supplementary_cubes.append(supplementary_cube) output_file = _get_output_file(self.facets, self.session.preproc_dir) @@ -704,7 +700,7 @@ def _load_with_callback(self, callback): return cubes[0] - def _load(self, callback) -> Cube: + def _load(self) -> Cube: """Load self.files into an iris cube and return it.""" if not self.files: lines = [ @@ -731,7 +727,6 @@ def _load(self, callback) -> Cube: **self.facets, } settings['load'] = { - 'callback': callback, 'ignore_warnings': get_ignored_warnings( self.facets['project'], 'load' ), diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 0cabc5d481..36d8cc4958 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -482,10 +482,7 @@ def apply(self, step: str, debug: bool = False): def cubes(self): """Cubes.""" if self._cubes is None: - callback = self.settings.get('load', {}).get('callback') - self._cubes = [ - ds._load_with_callback(callback) for ds in self.datasets - ] + self._cubes = [ds.load() for ds in self.datasets] return self._cubes @cubes.setter diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 62eb52853e..596b3b331a 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -6,7 +6,6 @@ import os import shutil import warnings -from collections.abc import Callable from itertools import groupby from pathlib import Path from typing import Optional @@ -96,7 +95,7 @@ def _get_attr_from_field_coord(ncfield, coord_name, attr): return None -def concatenate_callback(raw_cube, field, _): +def _load_callback(raw_cube, field, _): """Use this callback to fix anything Iris tries to break.""" # Remove attributes that cause issues with merging and concatenation _delete_attributes( @@ -122,7 +121,6 @@ def _delete_attributes(iris_object, atts): def load( file: str | Path, - callback: Optional[Callable] = None, ignore_warnings: Optional[list[dict]] = None, ) -> CubeList: """Load iris cubes from string or Path objects. @@ -131,11 +129,6 @@ def load( ---------- file: File to be loaded. Could be string or POSIX Path object. - callback: - Callback function passed to :func:`iris.load_raw`. - - .. deprecated:: 2.8.0 - This argument will be removed in 2.10.0. ignore_warnings: Keyword arguments passed to :func:`warnings.filterwarnings` used to ignore warnings issued by :func:`iris.load_raw`. Each list element @@ -152,14 +145,6 @@ def load( Cubes are empty. """ - if not (callback is None or callback == 'default'): - msg = ("The argument `callback` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in " - "version 2.10.0.") - warnings.warn(msg, ESMValCoreDeprecationWarning) - if callback == 'default': - callback = concatenate_callback - file = Path(file) logger.debug("Loading:\n%s", file) @@ -191,7 +176,7 @@ def load( # warnings.filterwarnings # (see https://github.com/SciTools/cf-units/issues/240) with suppress_errors(): - raw_cubes = iris.load_raw(file, callback=callback) + raw_cubes = iris.load_raw(file, callback=_load_callback) logger.debug("Done with loading %s", file) if not raw_cubes: diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index 99247bdc5a..c0ad0bba88 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -10,7 +10,7 @@ from iris.coords import DimCoord from iris.cube import Cube, CubeList -from esmvalcore.preprocessor._io import concatenate_callback, load +from esmvalcore.preprocessor._io import load def _create_sample_cube(): @@ -60,7 +60,7 @@ def test_callback_remove_attributes(self): cube.attributes[attr] = attr self._save_cube(cube) for temp_file in self.temp_files: - cubes = load(temp_file, callback='default') + cubes = load(temp_file) cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) @@ -79,7 +79,7 @@ def test_callback_remove_attributes_from_coords(self): coord.attributes[attr] = attr self._save_cube(cube) for temp_file in self.temp_files: - cubes = load(temp_file, callback='default') + cubes = load(temp_file) cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) @@ -94,7 +94,7 @@ def test_callback_fix_lat_units(self): cube = _create_sample_cube() temp_file = self._save_cube(cube) - cubes = load(temp_file, callback=concatenate_callback) + cubes = load(temp_file) cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) diff --git a/tests/integration/preprocessor/test_preprocessing_task.py b/tests/integration/preprocessor/test_preprocessing_task.py index 81f980c9e8..3b33e8f44e 100644 --- a/tests/integration/preprocessor/test_preprocessing_task.py +++ b/tests/integration/preprocessor/test_preprocessing_task.py @@ -16,7 +16,7 @@ def test_load_save_task(tmp_path): iris.save(cube, in_file) dataset = Dataset(short_name='tas') dataset.files = [in_file] - dataset._load_with_callback = lambda _: cube.copy() + dataset.load = lambda: cube.copy() # Create task task = PreprocessingTask([ @@ -57,11 +57,11 @@ def test_load_save_and_other_task(tmp_path, monkeypatch): dataset1 = Dataset(short_name='tas', dataset='dataset1') dataset1.files = [file1] - dataset1._load_with_callback = lambda _: in_cube.copy() + dataset1.load = lambda: in_cube.copy() dataset2 = Dataset(short_name='tas', dataset='dataset1') dataset2.files = [file2] - dataset2._load_with_callback = lambda _: in_cube.copy() + dataset2.load = lambda: in_cube.copy() # Create some mock preprocessor functions and patch # `esmvalcore.preprocessor` so it uses them. diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index ba53388ddf..6090e61c1c 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -82,7 +82,6 @@ ) DEFAULT_PREPROCESSOR_STEPS = ( - 'load', 'remove_supplementary_variables', 'save', ) @@ -106,9 +105,6 @@ def create_test_file(filename, tracking_id=None): def _get_default_settings_for_chl(save_filename): """Get default preprocessor settings for chl.""" defaults = { - 'load': { - 'callback': 'default' - }, 'remove_supplementary_variables': {}, 'save': { 'compress': False, @@ -557,9 +553,6 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, session): assert preproc_dir.startswith(str(tmp_path)) defaults = { - 'load': { - 'callback': 'default' - }, 'remove_supplementary_variables': {}, 'save': { 'compress': False, diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index a437221a0e..498dd606d9 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -541,7 +541,6 @@ def test_get_default_settings(mocker): settings = _recipe._get_default_settings(dataset) assert settings == { - 'load': {'callback': 'default'}, 'remove_supplementary_variables': {}, 'save': {'compress': False, 'alias': 'sic'}, } diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 56baaa60d8..1a0114e8c9 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1684,7 +1684,6 @@ def mock_preprocess(items, step, input_files, output_file, debug, load_args = { 'load': { - 'callback': 'default', 'ignore_warnings': None, }, 'fix_file': { From dcc9645dd12778541e24388cb48b395d44247143 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 14:27:22 +0200 Subject: [PATCH 10/72] Reduce the size of the dask graph created by the ``anomalies`` preprocessor function (#2200) Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_time.py | 41 +++++++++++++--------- tests/unit/preprocessor/_time/test_time.py | 4 ++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index e49afd4785..6c363379e5 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -12,6 +12,7 @@ from warnings import filterwarnings import dask.array as da +import dask.config import iris import iris.coord_categorisation import iris.exceptions @@ -738,7 +739,6 @@ def climate_statistics( ------- iris.cube.Cube Climate statistics cube. - """ original_dtype = cube.dtype period = period.lower() @@ -820,7 +820,6 @@ def anomalies( ------- iris.cube.Cube Anomalies cube. - """ if reference is None: reference_cube = cube @@ -862,23 +861,31 @@ def anomalies( return cube -def _compute_anomalies(cube, reference, period, seasons): +def _compute_anomalies( + cube: Cube, + reference: Cube, + period: str, + seasons: Iterable[str], +): cube_coord = _get_period_coord(cube, period, seasons) ref_coord = _get_period_coord(reference, period, seasons) - - data = cube.core_data() - cube_time = cube.coord('time') - ref = {} - for ref_slice in reference.slices_over(ref_coord): - ref[ref_slice.coord(ref_coord).points[0]] = ref_slice.core_data() - - cube_coord_dim = cube.coord_dims(cube_coord)[0] - slicer = [slice(None)] * len(data.shape) - new_data = [] - for i in range(cube_time.shape[0]): - slicer[cube_coord_dim] = i - new_data.append(data[tuple(slicer)] - ref[cube_coord.points[i]]) - data = da.stack(new_data, axis=cube_coord_dim) + indices = np.empty_like(cube_coord.points, dtype=np.int32) + for idx, point in enumerate(ref_coord.points): + indices = np.where(cube_coord.points == point, idx, indices) + ref_data = reference.core_data() + axis, = cube.coord_dims(cube_coord) + if cube.has_lazy_data() and reference.has_lazy_data(): + # Rechunk reference data because iris.cube.Cube.aggregate_by, used to + # compute the reference, produces very small chunks. + # https://github.com/SciTools/iris/issues/5455 + ref_chunks = tuple( + -1 if i == axis else chunk + for i, chunk in enumerate(cube.lazy_data().chunks) + ) + ref_data = ref_data.rechunk(ref_chunks) + with dask.config.set({"array.slicing.split_large_chunks": True}): + ref_data_broadcast = da.take(ref_data, indices=indices, axis=axis) + data = cube.core_data() - ref_data_broadcast cube = cube.copy(data) cube.remove_coord(cube_coord) return cube diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index b847a2972f..26b8ebf351 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import List, Tuple +import dask.array as da import iris import iris.coord_categorisation import iris.coords @@ -1658,8 +1659,9 @@ def make_map_data(number_years=2): standard_name='longitude', ) data = np.array([[0, 1], [1, 0]]) * times[:, None, None] + chunks = (int(data.shape[0] / 2), 1, 2) cube = iris.cube.Cube( - data, + da.asarray(data, chunks=chunks), dim_coords_and_dims=[(time, 0), (lat, 1), (lon, 2)], ) return cube From 925fce3c52c257ec98ececc2e5d421561e56309a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 14:53:39 +0200 Subject: [PATCH 11/72] Remove the deprecated option ``use_legacy_supplementaries`` (#2202) --- doc/recipe/preprocessor.rst | 128 --- esmvalcore/_recipe/recipe.py | 219 +--- esmvalcore/_recipe/to_datasets.py | 15 +- esmvalcore/config/_config_validators.py | 27 - esmvalcore/preprocessor/__init__.py | 4 - .../preprocessor/_supplementary_vars.py | 148 --- tests/integration/conftest.py | 89 +- .../test_add_fx_variables.py | 296 ------ .../test_add_supplementary_variables.py | 15 +- tests/integration/recipe/test_recipe.py | 976 +----------------- tests/integration/test_deprecated_config.py | 62 -- tests/unit/recipe/test_recipe.py | 50 - tests/unit/test_dataset.py | 21 +- 13 files changed, 109 insertions(+), 1941 deletions(-) delete mode 100644 tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 19020ba8f8..26c50df715 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -300,134 +300,6 @@ and cell measure (``areacella``), but do not use ``areacella`` for dataset timerange: '1990/2000' scripts: null - -.. _`Fx variables as cell measures or ancillary variables`: - -Legacy method of specifying supplementary variables ---------------------------------------------------- - -.. deprecated:: 2.8.0 - The legacy method of specifying supplementary variables is deprecated and will - be removed in version 2.10.0. - To upgrade, remove all occurrences of ``fx_variables`` from your recipes and - rely on automatically defining the supplementary variables based on the - requirement of the preprocessor functions or specify them using the methods - described above. - To keep using the legacy behaviour until v2.10.0, set - ``use_legacy_supplementaries: true`` in the :ref:`user configuration file` or - run the tool with the flag ``--use-legacy-supplementaries=True``. - -Prior to version 2.8.0 of the tool, the supplementary variables could not be -defined at the variable or dataset level in the recipe, but could only be -defined in the preprocessor function that uses them using the ``fx_variables`` -argument. -This does not work well because in practice different datasets store their -supplementary variables under different facets. -For example, one dataset might only provide the ``areacella`` variable under the -``1pctCO2`` experiment while another one might only provide it for the -``historical`` experiment. -This forced the user to define a preprocessor per dataset, which was -inconvenient. - -============================================================== ===================== -Preprocessor Default fx variables -============================================================== ===================== -:ref:`area_statistics` ``areacella``, ``areacello`` -:ref:`mask_landsea` ``sftlf``, ``sftof`` -:ref:`mask_landseaice` ``sftgif`` -:ref:`volume_statistics` ``volcello`` -:ref:`weighting_landsea_fraction` ``sftlf``, ``sftof`` -============================================================== ===================== - -If the option ``fx_variables`` is not explicitly specified for these -preprocessors, the default fx variables in the second column are automatically -used. If given, the ``fx_variables`` argument specifies the fx variables that -the user wishes to input to the corresponding preprocessor function. The user -may specify these by simply adding the names of the variables, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - volcello: - -or by additionally specifying further keys that are used to define the fx -datasets, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - mip: Ofx - exp: piControl - volcello: - mip: Omon - -This might be useful to select fx files from a specific ``mip`` table or from a -specific ``exp`` in case not all experiments provide the fx variable. - -Alternatively, the ``fx_variables`` argument can also be specified as a list: - -.. code-block:: yaml - - fx_variables: ['areacello', 'volcello'] - -or as a list of dictionaries: - -.. code-block:: yaml - - fx_variables: [{'short_name': 'areacello', 'mip': 'Ofx', 'exp': 'piControl'}, {'short_name': 'volcello', 'mip': 'Omon'}] - -The recipe parser will automatically find the data files that are associated -with these variables and pass them to the function for loading and processing. - -If ``mip`` is not given, ESMValCore will search for the fx variable in all -available tables of the specified project. - -.. warning:: - Some fx variables exist in more than one table (e.g., ``volcello`` exists in - CMIP6's ``Odec``, ``Ofx``, ``Omon``, and ``Oyr`` tables; ``sftgif`` exists - in CMIP6's ``fx``, ``IyrAnt`` and ``IyrGre``, and ``LImon`` tables). If (for - a given dataset) fx files are found in more than one table, ``mip`` needs to - be specified, otherwise an error is raised. - -.. note:: - To explicitly **not** use any fx variables in a preprocessor, use - ``fx_variables: null``. While some of the preprocessors mentioned above do - work without fx variables (e.g., ``area_statistics`` or ``mask_landsea`` - with datasets that have regular latitude/longitude grids), using this option - is **not** recommended. - -Internally, the required ``fx_variables`` are automatically loaded by the -preprocessor step ``add_fx_variables`` which also checks them against CMOR -standards and adds them either as ``cell_measure`` (see `CF conventions on cell -measures -`_ -and :class:`iris.coords.CellMeasure`) or ``ancillary_variable`` (see `CF -conventions on ancillary variables -`_ -and :class:`iris.coords.AncillaryVariable`) inside the cube data. This ensures -that the defined preprocessor chain is applied to both ``variables`` and -``fx_variables``. - -Note that when calling steps that require ``fx_variables`` inside diagnostic -scripts, the variables are expected to contain the required ``cell_measures`` or -``Fx variables as cell measures or ancillary variables``. If missing, they can be added using the following functions: - -.. code-block:: - - from esmvalcore.preprocessor import (add_cell_measure, add_ancillary_variable) - - cube_with_area_measure = add_cell_measure(cube, area_cube, 'area') - - cube_with_volume_measure = add_cell_measure(cube, volume_cube, 'volume) - - cube_with_ancillary_sftlf = add_ancillary_variable(cube, sftlf_cube) - - cube_with_ancillary_sftgif = add_ancillary_variable(cube, sftgif_cube) - - Details on the arguments needed for each step can be found in the following sections. - .. _Vertical interpolation: Vertical interpolation diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 6f003e17e0..3bcc45720d 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -9,7 +9,6 @@ from copy import deepcopy from itertools import groupby from pathlib import Path -from pprint import pformat from typing import Any, Dict, Iterable, Sequence import yaml @@ -17,13 +16,10 @@ from esmvalcore import __version__, esgf from esmvalcore._provenance import get_recipe_provenance from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet -from esmvalcore.cmor.table import CMOR_TABLES, _update_cmor_facets -from esmvalcore.config import CFG -from esmvalcore.config._config import TASKSEP, get_project_config +from esmvalcore.config._config import TASKSEP from esmvalcore.config._diagnostics import TAGS from esmvalcore.dataset import Dataset from esmvalcore.exceptions import ( - ESMValCoreDeprecationWarning, InputFilesNotFound, RecipeError, ) @@ -50,10 +46,6 @@ get_reference_levels, parse_cell_spec, ) -from esmvalcore.preprocessor._supplementary_vars import ( - PREPROCESSOR_SUPPLEMENTARIES, -) -from esmvalcore.typing import Facets from . import check from .from_datasets import datasets_to_recipe @@ -232,177 +224,6 @@ def _get_default_settings(dataset): return settings -def _guess_fx_mip(facets: dict, dataset: Dataset): - """Search mip for fx variable.""" - project = facets.get('project', dataset.facets['project']) - # check if project in config-developer - get_project_config(project) - - tables = CMOR_TABLES[project].tables - - # Get all mips that offer that specific fx variable - mips_with_fx_var = [] - for mip in tables: - if facets['short_name'] in tables[mip]: - mips_with_fx_var.append(mip) - - # List is empty -> no table includes the fx variable - if not mips_with_fx_var: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' not available " - f"in any CMOR table for '{project}'") - - # Iterate through all possible mips and check if files are available; in - # case of ambiguity raise an error - fx_files_for_mips = {} - for mip in mips_with_fx_var: - logger.debug("For fx variable '%s', found table '%s'", - facets['short_name'], mip) - fx_dataset = dataset.copy(**facets) - fx_dataset.supplementaries = [] - fx_dataset.set_facet('mip', mip) - fx_dataset.facets.pop('timerange', None) - fx_files = fx_dataset.files - if fx_files: - logger.debug("Found fx variables '%s':\n%s", facets['short_name'], - pformat(fx_files)) - fx_files_for_mips[mip] = fx_files - - # Dict contains more than one element -> ambiguity - if len(fx_files_for_mips) > 1: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' for dataset " - f"'{dataset.facets['dataset']}' of project '{project}' is " - f"available in more than one CMOR MIP table for " - f"'{project}': {sorted(fx_files_for_mips)}") - - # Dict is empty -> no files found -> handled at later stage - if not fx_files_for_mips: - return mips_with_fx_var[0] - - # Dict contains one element -> ok - mip = list(fx_files_for_mips)[0] - return mip - - -def _set_default_preproc_fx_variables( - dataset: Dataset, - settings: PreprocessorSettings, -) -> None: - """Update `fx_variables` key in preprocessor settings with defaults.""" - default_fx = { - 'area_statistics': { - 'areacella': None, - }, - 'mask_landsea': { - 'sftlf': None, - }, - 'mask_landseaice': { - 'sftgif': None, - }, - 'volume_statistics': { - 'volcello': None, - }, - 'weighting_landsea_fraction': { - 'sftlf': None, - }, - } - if dataset.facets['project'] != 'obs4MIPs': - default_fx['area_statistics']['areacello'] = None - default_fx['mask_landsea']['sftof'] = None - default_fx['weighting_landsea_fraction']['sftof'] = None - - for step, fx_variables in default_fx.items(): - if step in settings and 'fx_variables' not in settings[step]: - settings[step]['fx_variables'] = fx_variables - - -def _get_supplementaries_from_fx_variables( - settings: PreprocessorSettings -) -> list[Facets]: - """Read supplementary facets from `fx_variables` in preprocessor.""" - supplementaries = [] - for step, kwargs in settings.items(): - allowed = PREPROCESSOR_SUPPLEMENTARIES.get(step, - {}).get('variables', []) - if fx_variables := kwargs.get('fx_variables'): - - if isinstance(fx_variables, list): - result: dict[str, Facets] = {} - for fx_variable in fx_variables: - if isinstance(fx_variable, str): - # Legacy legacy method of specifying fx variable - short_name = fx_variable - result[short_name] = {} - elif isinstance(fx_variable, dict): - short_name = fx_variable['short_name'] - result[short_name] = fx_variable - fx_variables = result - - for short_name, facets in fx_variables.items(): - if short_name not in allowed: - raise RecipeError( - f"Preprocessor function '{step}' does not support " - f"supplementary variable '{short_name}'") - if facets is None: - facets = {} - facets['short_name'] = short_name - supplementaries.append(facets) - - return supplementaries - - -def _get_legacy_supplementary_facets( - dataset: Dataset, - settings: PreprocessorSettings, -) -> list[Facets]: - """Load the supplementary dataset facets from the preprocessor settings.""" - # First update `fx_variables` in preprocessor settings with defaults - _set_default_preproc_fx_variables(dataset, settings) - - supplementaries = _get_supplementaries_from_fx_variables(settings) - - # Guess the ensemble and mip if they is not specified - for facets in supplementaries: - if 'ensemble' not in facets and dataset.facets['project'] == 'CMIP5': - facets['ensemble'] = 'r0i0p0' - if 'mip' not in facets: - facets['mip'] = _guess_fx_mip(facets, dataset) - return supplementaries - - -def _add_legacy_supplementary_datasets(dataset: Dataset, settings): - """Update fx settings depending on the needed method.""" - if not dataset.session['use_legacy_supplementaries']: - return - if dataset.supplementaries: - # Supplementaries have been defined in the recipe. - # Just remove any skipped supplementaries (they have been kept so we - # know that supplementaries have been defined in the recipe). - dataset.supplementaries = [ - ds for ds in dataset.supplementaries - if not ds.facets.get('skip', False) - ] - return - - logger.debug("Using legacy method to add supplementaries to %s", dataset) - - legacy_ds = dataset.copy() - for facets in _get_legacy_supplementary_facets(dataset, settings): - legacy_ds.add_supplementary(**facets) - - for supplementary_ds in legacy_ds.supplementaries: - _update_cmor_facets(supplementary_ds.facets, override=True) - if supplementary_ds.files: - dataset.supplementaries.append(supplementary_ds) - - dataset._fix_fx_exp() - - # Remove preprocessor keyword argument `fx_variables` - for kwargs in settings.values(): - kwargs.pop('fx_variables', None) - - def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { @@ -684,7 +505,6 @@ def _get_preprocessor_products( _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(dataset.facets, settings) _update_preproc_functions(settings, dataset, datasets, missing_vars) - _add_legacy_supplementary_datasets(dataset, settings) check.preprocessor_supplementaries(dataset, settings) input_datasets = _get_input_datasets(dataset) missing = _check_input_files(input_datasets) @@ -890,7 +710,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._preprocessors = raw_recipe.get('preprocessors', {}) if 'default' not in self._preprocessors: self._preprocessors['default'] = {} - self._set_use_legacy_supplementaries() self.datasets = Dataset.from_recipe(recipe_file, session) self.diagnostics = self._initialize_diagnostics( raw_recipe['diagnostics']) @@ -902,42 +721,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._log_recipe_errors(exc) raise - def _set_use_legacy_supplementaries(self): - """Automatically determine if legacy supplementaries are used.""" - names = set() - steps = set() - for name, profile in self._preprocessors.items(): - for step, kwargs in profile.items(): - if isinstance(kwargs, dict) and 'fx_variables' in kwargs: - names.add(name) - steps.add(step) - if self.session['use_legacy_supplementaries'] is False: - kwargs.pop('fx_variables') - if names: - warnings.warn( - ESMValCoreDeprecationWarning( - "Encountered 'fx_variables' argument in preprocessor(s) " - f"{sorted(names)}, function(s) {sorted(steps)}. The " - "'fx_variables' argument is deprecated and will stop " - "working in v2.10. Please remove it and if automatic " - "definition of supplementary variables does not work " - "correctly, specify the supplementary variables in the " - "recipe as described in https://docs.esmvaltool.org/" - "projects/esmvalcore/en/latest/recipe/preprocessor.html" - "#ancillary-variables-and-cell-measures")) - if self.session['use_legacy_supplementaries'] is None: - logger.info("Running with --use-legacy-supplementaries=True") - self.session['use_legacy_supplementaries'] = True - - # Also adapt the global config if necessary because it is used to check - # if mismatching shapes should be ignored when attaching - # supplementary variables in `esmvalcore.preprocessor. - # _supplementary_vars.add_supplementary_variables` to avoid having to - # introduce a new function argument that is immediately deprecated. - session_use_legacy_supp = self.session['use_legacy_supplementaries'] - if session_use_legacy_supp is not None: - CFG['use_legacy_supplementaries'] = session_use_legacy_supp - def _log_recipe_errors(self, exc): """Log a message with recipe errors.""" logger.error(exc.message) diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 06423cbed6..56d9d44221 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -288,14 +288,13 @@ def _get_dataset_facets_from_recipe( ), ) - if not session['use_legacy_supplementaries']: - preprocessor = facets.get('preprocessor', 'default') - settings = profiles.get(preprocessor, {}) - _append_missing_supplementaries(supplementaries, facets, settings) - supplementaries = [ - facets for facets in supplementaries - if not facets.pop('skip', False) - ] + preprocessor = facets.get('preprocessor', 'default') + settings = profiles.get(preprocessor, {}) + _append_missing_supplementaries(supplementaries, facets, settings) + supplementaries = [ + facets for facets in supplementaries + if not facets.pop('skip', False) + ] return facets, supplementaries diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 736a6ba689..8f5e47375a 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -297,7 +297,6 @@ def validate_diagnostics( 'run_diagnostic': validate_bool, 'save_intermediary_cubes': validate_bool, 'search_esgf': validate_search_esgf, - 'use_legacy_supplementaries': validate_bool_or_none, # From CLI 'check_level': validate_check_level, @@ -372,38 +371,12 @@ def deprecate_offline( validated_config['search_esgf'] = 'when_missing' -def deprecate_use_legacy_supplementaries( - validated_config: ValidatedConfig, - value: Any, - validated_value: Any, -) -> None: - """Deprecate ``use_legacy_supplementaries`` option. - - Parameters - ---------- - validated_config: ValidatedConfig - ``ValidatedConfig`` instance which will be modified in place. - value: Any - Raw input value for ``use_legacy_supplementaries`` option. - validated_value: Any - Validated value for ``use_legacy_supplementaries`` option. - - """ - option = 'use_legacy_supplementaries' - deprecated_version = '2.8.0' - remove_version = '2.10.0' - more_info = '' - _handle_deprecation(option, deprecated_version, remove_version, more_info) - - _deprecators: dict[str, Callable] = { 'offline': deprecate_offline, - 'use_legacy_supplementaries': deprecate_use_legacy_supplementaries, } # Default values for deprecated options _deprecated_options_defaults: dict[str, Any] = { 'offline': True, - 'use_legacy_supplementaries': None, } diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 36d8cc4958..d66592a5b9 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -57,9 +57,7 @@ ) from ._rolling_window import rolling_window_statistics from ._supplementary_vars import ( - add_fx_variables, add_supplementary_variables, - remove_fx_variables, remove_supplementary_variables, ) from ._time import ( @@ -110,7 +108,6 @@ 'fix_data', 'cmor_check_data', # Attach ancillary variables and cell measures - 'add_fx_variables', 'add_supplementary_variables', # Derive variable 'derive', @@ -189,7 +186,6 @@ 'bias', # Remove supplementary variables from cube 'remove_supplementary_variables', - 'remove_fx_variables', # Save to file 'save', 'cleanup', diff --git a/esmvalcore/preprocessor/_supplementary_vars.py b/esmvalcore/preprocessor/_supplementary_vars.py index 8813735098..d5cb0e2d31 100644 --- a/esmvalcore/preprocessor/_supplementary_vars.py +++ b/esmvalcore/preprocessor/_supplementary_vars.py @@ -1,22 +1,11 @@ """Preprocessor functions for ancillary variables and cell measures.""" import logging -import warnings -from pathlib import Path from typing import Iterable -import dask.array as da import iris.coords import iris.cube -from esmvalcore.cmor.check import cmor_check_data, cmor_check_metadata -from esmvalcore.cmor.fix import fix_data, fix_metadata -from esmvalcore.config import CFG -from esmvalcore.config._config import get_ignored_warnings -from esmvalcore.exceptions import ESMValCoreDeprecationWarning -from esmvalcore.preprocessor._io import concatenate, load -from esmvalcore.preprocessor._time import clip_timerange - logger = logging.getLogger(__name__) PREPROCESSOR_SUPPLEMENTARIES = {} @@ -50,62 +39,6 @@ def wrapper(func): return wrapper -def _load_fx(var_cube, fx_info, check_level): - """Load and CMOR-check fx variables.""" - fx_cubes = iris.cube.CubeList() - - project = fx_info['project'] - mip = fx_info['mip'] - short_name = fx_info['short_name'] - freq = fx_info['frequency'] - - for fx_file in fx_info['filename']: - ignored_warnings = get_ignored_warnings(project, 'load') - loaded_cube = load(fx_file, ignore_warnings=ignored_warnings) - loaded_cube = fix_metadata(loaded_cube, - check_level=check_level, - **fx_info) - fx_cubes.append(loaded_cube[0]) - - fx_cube = concatenate(fx_cubes) - - if freq != 'fx': - fx_cube = clip_timerange(fx_cube, fx_info['timerange']) - - if not _is_fx_broadcastable(fx_cube, var_cube): - return None - - fx_cube = cmor_check_metadata(fx_cube, - cmor_table=project, - mip=mip, - short_name=short_name, - frequency=freq, - check_level=check_level) - - fx_cube = fix_data(fx_cube, check_level=check_level, **fx_info) - - fx_cube = cmor_check_data(fx_cube, - cmor_table=project, - mip=mip, - short_name=fx_cube.var_name, - frequency=freq, - check_level=check_level) - - return fx_cube - - -def _is_fx_broadcastable(fx_cube, cube): - try: - da.broadcast_to(fx_cube.core_data(), cube.shape) - except ValueError as exc: - logger.debug( - "Dimensions of %s and %s cubes do not match. " - "Discarding use of fx_variable: %s", cube.var_name, - fx_cube.var_name, exc) - return False - return True - - def add_cell_measure(cube, cell_measure_cube, measure): """Add a cube as a cell_measure in the cube containing the data. @@ -172,56 +105,6 @@ def add_ancillary_variable(cube, ancillary_cube): ancillary_cube.var_name, cube.var_name) -def add_fx_variables(cube, fx_variables, check_level): - """Load requested fx files, check with CMOR standards and add the fx - variables as cell measures or ancillary variables in the cube containing - the data. - - .. deprecated:: 2.8.0 - This function is deprecated and will be removed in version 2.10.0. - Please use a :class:`esmvalcore.dataset.Dataset` or - :func:`esmvalcore.preprocessor.add_supplementary_variables` - instead. - - Parameters - ---------- - cube: iris.cube.Cube - Iris cube with input data. - fx_variables: dict - Dictionary with fx_variable information. - check_level: CheckLevels - Level of strictness of the checks. - - Returns - ------- - iris.cube.Cube - Cube with added cell measures or ancillary variables. - """ - msg = ( - "The function `add_fx_variables` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in " - "version 2.10.0. Use a `esmvalcore.dataset.Dataset` or the function " - "`add_supplementary_variables` instead.") - warnings.warn(msg, ESMValCoreDeprecationWarning) - if not fx_variables: - return cube - fx_cubes = [] - for fx_info in fx_variables.values(): - if not fx_info: - continue - if isinstance(fx_info['filename'], (str, Path)): - fx_info['filename'] = [fx_info['filename']] - fx_cube = _load_fx(cube, fx_info, check_level) - - if fx_cube is None: - continue - - fx_cubes.append(fx_cube) - - add_supplementary_variables(cube, fx_cubes) - return cube - - def add_supplementary_variables( cube: iris.cube.Cube, supplementary_cubes: Iterable[iris.cube.Cube], @@ -246,9 +129,6 @@ def add_supplementary_variables( 'volcello': 'volume' } for supplementary_cube in supplementary_cubes: - if (CFG['use_legacy_supplementaries'] - and not _is_fx_broadcastable(supplementary_cube, cube)): - continue if supplementary_cube.var_name in measure_names: measure_name = measure_names[supplementary_cube.var_name] add_cell_measure(cube, supplementary_cube, measure_name) @@ -280,31 +160,3 @@ def remove_supplementary_variables(cube: iris.cube.Cube): for variable in cube.ancillary_variables(): cube.remove_ancillary_variable(variable) return cube - - -def remove_fx_variables(cube): - """Remove fx variables present as cell measures or ancillary variables in - the cube containing the data. - - .. deprecated:: 2.8.0 - This function is deprecated and will be removed in version 2.10.0. - Please use - :func:`esmvalcore.preprocessor.remove_supplementary_variables` - instead. - - Parameters - ---------- - cube: iris.cube.Cube - Iris cube with data and cell measures or ancillary variables. - - Returns - ------- - iris.cube.Cube - Cube without cell measures or ancillary variables. - """ - msg = ("The function `remove_fx_variables` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in " - "version 2.10.0. Use the function `remove_supplementary_variables` " - "instead.") - warnings.warn(msg, ESMValCoreDeprecationWarning) - return remove_supplementary_variables(cube) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1dbb948940..2771db32c0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,25 +5,24 @@ import pytest import esmvalcore.local -from esmvalcore.config import CFG, _config +from esmvalcore.config import CFG from esmvalcore.config._config_object import CFG_DEFAULT +from esmvalcore.local import ( + LocalFile, + _replace_tags, + _select_drs, + _select_files, +) @pytest.fixture -def session(tmp_path, monkeypatch): +def session(tmp_path: Path, monkeypatch): + CFG.clear() + CFG.update(CFG_DEFAULT) + monkeypatch.setitem(CFG, 'rootpath', {'default': str(tmp_path)}) + session = CFG.start_session('recipe_test') - session.clear() - session.update(CFG_DEFAULT) session['output_dir'] = tmp_path / 'esmvaltool_output' - - # The patched_datafinder fixture does not return the correct input - # directory structure, so make sure it is set to flat for every project - monkeypatch.setitem(CFG, 'drs', {}) - for project in _config.CFG: - monkeypatch.setitem(_config.CFG[project]['input_dir'], 'default', '/') - # The patched datafinder fixture does not return any facets, so automatic - # supplementary definition does not work with it. - session['use_legacy_supplementaries'] = True return session @@ -40,8 +39,10 @@ def create_test_file(filename, tracking_id=None): iris.save(cube, filename) -def _get_filenames(root_path, filename, tracking_id): - filename = Path(filename).name +def _get_files(root_path, facets, tracking_id): + file_template = _select_drs('input_file', facets['project']) + file_globs = _replace_tags(file_template, facets) + filename = Path(file_globs[0]).name filename = str(root_path / 'input' / filename) filenames = [] if filename.endswith('[_.]*nc'): @@ -50,23 +51,37 @@ def _get_filenames(root_path, filename, tracking_id): filename = filename.replace('[_.]*nc', '_*.nc') if filename.endswith('*.nc'): filename = filename[:-len('*.nc')] + '_' - intervals = [ - '1990_1999', - '2000_2009', - '2010_2019', - ] + if facets['frequency'] == 'fx': + intervals = [''] + else: + intervals = [ + '1990_1999', + '2000_2009', + '2010_2019', + ] for interval in intervals: filenames.append(filename + interval + '.nc') else: filenames.append(filename) + if 'timerange' in facets: + filenames = _select_files(filenames, facets['timerange']) + for filename in filenames: create_test_file(filename, next(tracking_id)) - return filenames + + files = [] + for filename in filenames: + file = LocalFile(filename) + file.facets = facets + files.append(file) + + return files, file_globs @pytest.fixture def patched_datafinder(tmp_path, monkeypatch): + def tracking_ids(i=0): while True: yield i @@ -74,14 +89,18 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def glob(file_glob): - return _get_filenames(tmp_path, file_glob, tracking_id) + def find_files(*, debug: bool = False, **facets): + files, file_globs = _get_files(tmp_path, facets, tracking_id) + if debug: + return files, file_globs + return files - monkeypatch.setattr(esmvalcore.local, 'glob', glob) + monkeypatch.setattr(esmvalcore.local, 'find_files', find_files) @pytest.fixture def patched_failing_datafinder(tmp_path, monkeypatch): + def tracking_ids(i=0): while True: yield i @@ -89,16 +108,12 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def glob(filename): - # Fail for specified fx variables - if 'fx_' in filename: - return [] - if 'sftlf' in filename: - return [] - if 'IyrAnt_' in filename: - return [] - if 'IyrGre_' in filename: - return [] - return _get_filenames(tmp_path, filename, tracking_id) - - monkeypatch.setattr(esmvalcore.local, 'glob', glob) + def find_files(*, debug: bool = False, **facets): + files, file_globs = _get_files(tmp_path, facets, tracking_id) + if 'fx' == facets['frequency']: + files = [] + if debug: + return files, file_globs + return files + + monkeypatch.setattr(esmvalcore.local, 'find_files', find_files) diff --git a/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py deleted file mode 100644 index 415a5849c4..0000000000 --- a/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Test add_fx_variables. - -Integration tests for the -:func:`esmvalcore.preprocessor._supplementary_vars` module. -""" -import logging - -import iris -import numpy as np -import pytest - -from esmvalcore.cmor.check import CheckLevels -from esmvalcore.preprocessor._supplementary_vars import ( - _is_fx_broadcastable, - add_ancillary_variable, - add_cell_measure, - add_fx_variables, - remove_fx_variables, -) -from esmvalcore.preprocessor._time import clip_timerange - -logger = logging.getLogger(__name__) - -SHAPES_TO_BROADCAST = [ - ((), (1, ), True), - ((), (10, 10), True), - ((1, ), (10, ), True), - ((1, ), (10, 10), True), - ((2, ), (10, ), False), - ((10, ), (), False), - ((10, ), (1, ), False), - ((10, ), (10, ), True), - ((10, ), (10, 10), True), - ((10, ), (7, 1), False), - ((10, ), (10, 7), False), - ((10, ), (7, 1, 10), True), - ((10, ), (7, 1, 1), False), - ((10, ), (7, 1, 7), False), - ((10, ), (7, 10, 7), False), - ((10, 1), (1, 1), False), - ((10, 1), (1, 100), False), - ((10, 1), (10, 7), True), - ((10, 12), (10, 1), False), - ((10, 1), (10, 12), True), - ((10, 12), (), False), - ((), (10, 12), True), - ((10, 12), (1, ), False), - ((1, ), (10, 12), True), - ((10, 12), (12, ), False), - ((10, 12), (1, 1), False), - ((1, 1), (10, 12), True), - ((10, 12), (1, 12), False), - ((1, 12), (10, 12), True), - ((10, 12), (10, 10, 1), False), - ((10, 12), (10, 12, 1), False), - ((10, 12), (10, 12, 12), False), - ((10, 12), (10, 10, 12), True)] - - -@pytest.mark.parametrize('shape_1,shape_2,out', SHAPES_TO_BROADCAST) -def test_shape_is_broadcastable(shape_1, shape_2, out): - """Test check if two shapes are broadcastable.""" - fx_cube = iris.cube.Cube(np.ones(shape_1)) - cube = iris.cube.Cube(np.ones(shape_2)) - is_broadcastable = _is_fx_broadcastable(fx_cube, cube) - assert is_broadcastable == out - - -class Test: - """Test class.""" - @pytest.fixture(autouse=True) - def setUp(self): - """Assemble a stock cube.""" - fx_area_data = np.ones((3, 3)) - fx_volume_data = np.ones((3, 3, 3)) - self.new_cube_data = np.empty((3, 3)) - self.new_cube_data[:] = 200. - self.new_cube_3D_data = np.empty((3, 3, 3)) - self.new_cube_3D_data[:] = 200. - crd_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) - self.lons = iris.coords.DimCoord([0, 1.5, 3], - standard_name='longitude', - bounds=[[0, 1], [1, 2], [2, 3]], - units='degrees_east', - coord_system=crd_sys) - self.lats = iris.coords.DimCoord([0, 1.5, 3], - standard_name='latitude', - bounds=[[0, 1], [1, 2], [2, 3]], - units='degrees_north', - coord_system=crd_sys) - self.depth = iris.coords.DimCoord([0, 1.5, 3], - standard_name='depth', - bounds=[[0, 1], [1, 2], [2, 3]], - units='m', - long_name='ocean depth coordinate') - self.monthly_times = iris.coords.DimCoord( - [15.5, 45, 74.5, 105, 135.5, 166, - 196.5, 227.5, 258, 288.5, 319, 349.5], - standard_name='time', - var_name='time', - bounds=[[0, 31], [31, 59], [59, 90], - [90, 120], [120, 151], [151, 181], - [181, 212], [212, 243], [243, 273], - [273, 304], [304, 334], [334, 365]], - units='days since 1950-01-01 00:00:00') - self.yearly_times = iris.coords.DimCoord( - [182.5, 547.5], - standard_name='time', - bounds=[[0, 365], [365, 730]], - units='days since 1950-01-01 00:00') - self.coords_spec = [(self.lats, 0), (self.lons, 1)] - self.fx_area = iris.cube.Cube(fx_area_data, - dim_coords_and_dims=self.coords_spec) - self.fx_volume = iris.cube.Cube(fx_volume_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2) - ]) - self.monthly_volume = iris.cube.Cube(np.ones((12, 3, 3, 3)), - dim_coords_and_dims=[ - (self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3) - ]) - - def test_add_cell_measure_area(self, tmp_path): - """Test add area fx variables as cell measures.""" - fx_vars = { - 'areacella': { - 'short_name': 'areacella', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx'}, - 'areacello': { - 'short_name': 'areacello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Ofx', - 'frequency': 'fx' - } - } - for fx_var in fx_vars: - self.fx_area.var_name = fx_var - self.fx_area.standard_name = 'cell_area' - self.fx_area.units = 'm2' - fx_file = str(tmp_path / f'{fx_var}.nc') - fx_vars[fx_var].update({'filename': fx_file}) - iris.save(self.fx_area, fx_file) - cube = iris.cube.Cube(self.new_cube_data, - dim_coords_and_dims=self.coords_spec) - cube = add_fx_variables( - cube, {fx_var: fx_vars[fx_var]}, CheckLevels.IGNORE) - assert cube.cell_measure(self.fx_area.standard_name) is not None - - def test_add_cell_measure_volume(self, tmp_path): - """Test add volume as cell measure.""" - fx_vars = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Ofx', - 'frequency': 'fx'} - } - self.fx_volume.var_name = 'volcello' - self.fx_volume.standard_name = 'ocean_volume' - self.fx_volume.units = 'm3' - fx_file = str(tmp_path / 'volcello.nc') - iris.save(self.fx_volume, fx_file) - fx_vars['volcello'].update({'filename': fx_file}) - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.cell_measure(self.fx_volume.standard_name) is not None - - def test_clip_volume_timerange(self, tmp_path): - """Test timerange is clipped in time dependent measures.""" - cell_measures = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Omon', - 'frequency': 'mon', - 'timerange': '195001/195003'} - } - self.monthly_volume.var_name = 'volcello' - self.monthly_volume.standard_name = 'ocean_volume' - self.monthly_volume.units = 'm3' - cell_measure_file = str(tmp_path / 'volcello.nc') - iris.save(self.monthly_volume, cell_measure_file) - cell_measures['volcello'].update( - {'filename': cell_measure_file}) - cube = iris.cube.Cube(np.ones((12, 3, 3, 3)), - dim_coords_and_dims=[ - (self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - cube = clip_timerange(cube, '195001/195003') - cube = add_fx_variables(cube, cell_measures, CheckLevels.IGNORE) - cell_measure = cube.cell_measure(self.fx_volume.standard_name) - assert cell_measure is not None - assert cell_measure.shape == (3, 3, 3, 3) - - def test_no_cell_measure(self): - """Test no cell measure is added.""" - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - cube = add_fx_variables(cube, {'areacello': None}, CheckLevels.IGNORE) - assert cube.cell_measures() == [] - - def test_add_ancillary_variables(self, tmp_path): - """Test invalid variable is not added as cell measure.""" - self.fx_area.var_name = 'sftlf' - self.fx_area.standard_name = "land_area_fraction" - self.fx_area.units = '%' - fx_file = str(tmp_path / f'{self.fx_area.var_name}.nc') - iris.save(self.fx_area, fx_file) - fx_vars = { - 'sftlf': { - 'short_name': 'sftlf', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': fx_file} - } - cube = iris.cube.Cube(self.new_cube_data, - dim_coords_and_dims=self.coords_spec) - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.ancillary_variable(self.fx_area.standard_name) is not None - - def test_wrong_shape(self, tmp_path): - """Test fx_variable is not added if it's not broadcastable to cube.""" - volume_data = np.ones((2, 3, 3, 3)) - volume_cube = iris.cube.Cube( - volume_data, - dim_coords_and_dims=[(self.yearly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - volume_cube.standard_name = 'ocean_volume' - volume_cube.var_name = 'volcello' - volume_cube.units = 'm3' - fx_file = str(tmp_path / f'{volume_cube.var_name}.nc') - iris.save(volume_cube, fx_file) - fx_vars = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Oyr', - 'frequency': 'yr', - 'filename': fx_file, - 'timerange': '1950/1951'} - } - data = np.ones((12, 3, 3, 3)) - cube = iris.cube.Cube( - data, - dim_coords_and_dims=[(self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - cube.var_name = 'thetao' - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.cell_measures() == [] - - def test_remove_fx_vars(self): - """Test fx_variables are removed from cube.""" - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[(self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - self.fx_area.var_name = 'areacella' - self.fx_area.standard_name = 'cell_area' - self.fx_area.units = 'm2' - add_cell_measure(cube, self.fx_area, measure='area') - assert cube.cell_measure(self.fx_area.standard_name) is not None - self.fx_area.var_name = 'sftlf' - self.fx_area.standard_name = "land_area_fraction" - self.fx_area.units = '%' - add_ancillary_variable(cube, self.fx_area) - assert cube.ancillary_variable(self.fx_area.standard_name) is not None - cube = remove_fx_variables(cube) - assert cube.cell_measures() == [] - assert cube.ancillary_variables() == [] diff --git a/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py index 9a3fe8169c..d4a9b0b217 100644 --- a/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py +++ b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py @@ -8,7 +8,6 @@ import numpy as np import pytest -import esmvalcore.config from esmvalcore.preprocessor._supplementary_vars import ( add_ancillary_variable, add_cell_measure, @@ -120,14 +119,8 @@ def test_add_supplementary_vars(self): cube = add_supplementary_variables(cube, [self.fx_area]) assert cube.ancillary_variable(self.fx_area.standard_name) is not None - @pytest.mark.parametrize('use_legacy_supplementaries', [True, False]) - def test_wrong_shape(self, use_legacy_supplementaries, monkeypatch): + def test_wrong_shape(self, monkeypatch): """Test variable is not added if it's not broadcastable to cube.""" - monkeypatch.setitem( - esmvalcore.config.CFG, - 'use_legacy_supplementaries', - use_legacy_supplementaries, - ) volume_data = np.ones((2, 3, 3, 3)) volume_cube = iris.cube.Cube( volume_data, @@ -146,12 +139,8 @@ def test_wrong_shape(self, use_legacy_supplementaries, monkeypatch): (self.lats, 2), (self.lons, 3)]) cube.var_name = 'thetao' - if use_legacy_supplementaries: + with pytest.raises(iris.exceptions.CannotAddError): add_supplementary_variables(cube, [volume_cube]) - assert cube.cell_measures() == [] - else: - with pytest.raises(iris.exceptions.CannotAddError): - add_supplementary_variables(cube, [volume_cube]) def test_remove_supplementary_vars(self): """Test supplementary variables are removed from cube.""" diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 6090e61c1c..88bde98f31 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -116,6 +116,7 @@ def _get_default_settings_for_chl(save_filename): @pytest.fixture def patched_tas_derivation(monkeypatch): + def get_required(short_name, _): if short_name != 'tas': assert False @@ -1818,6 +1819,9 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: sftlf + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -1836,16 +1840,8 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): assert settings['area_type'] == 'land' assert len(product.datasets) == 1 dataset = product.datasets[0] - short_names = { - ds.facets['short_name'] - for ds in dataset.supplementaries - } - if dataset.facets['project'] == 'obs4MIPs': - assert len(dataset.supplementaries) == 1 - assert {'sftlf'} == short_names - else: - assert len(dataset.supplementaries) == 2 - assert {'sftlf', 'sftof'} == short_names + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'sftlf' def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, @@ -1903,8 +1899,8 @@ def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, additional_datasets: - {dataset: CanESM2} - {dataset: GFDL-CM3} - - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, - tier: 1} + - {dataset: TEST, project: obs4MIPs, + supplementary_variables: [{short_name: sftlf, mip: fx}]} scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -1984,6 +1980,9 @@ def test_area_statistics(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: areacella + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -2002,14 +2001,8 @@ def test_area_statistics(tmp_path, patched_datafinder, session): assert settings['operator'] == 'mean' assert len(product.datasets) == 1 dataset = product.datasets[0] - short_names = { - ds.facets['short_name'] - for ds in dataset.supplementaries - } - if dataset.facets['project'] == 'obs4MIPs': - assert short_names == {'areacella'} - else: - assert short_names == {'areacella', 'areacello'} + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'areacella' def test_landmask(tmp_path, patched_datafinder, session): @@ -2034,6 +2027,9 @@ def test_landmask(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: sftlf + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -2052,223 +2048,8 @@ def test_landmask(tmp_path, patched_datafinder, session): assert settings['mask_out'] == 'sea' assert len(product.datasets) == 1 dataset = product.datasets[0] - if dataset.facets['project'] == 'obs4MIPs': - assert len(dataset.supplementaries) == 1 - else: - assert len(dataset.supplementaries) == 2 - - -def test_empty_fxvar_none(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: null - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -def test_empty_fxvar_list(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [] - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -def test_empty_fxvar_dict(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: {} - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -@pytest.mark.parametrize('content', [ - pytest.param(dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: - sftlf: - exp: piControl - mask_landseaice: - mask_out: sea - fx_variables: - sftgif: - exp: piControl - volume_statistics: - operator: mean - area_statistics: - operator: mean - fx_variables: - areacello: - mip: fx - exp: piControl - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """), - id='fx_variables_as_dict_of_dicts'), - pytest.param(dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [{'short_name': 'sftlf', 'exp': 'piControl'}] - mask_landseaice: - mask_out: sea - fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] - volume_statistics: - operator: mean - area_statistics: - operator: mean - fx_variables: [{'short_name': 'areacello', 'mip': 'fx', - 'exp': 'piControl'}] - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """), - id='fx_variables_as_list_of_dicts'), -]) -def test_user_defined_fxvar(tmp_path, patched_datafinder, session, content): - recipe = get_recipe(tmp_path, content, session) - - # Check custom fx variables - task = recipe.tasks.pop() - product = task.products.pop() - - # landsea - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert isinstance(dataset.supplementaries, list) - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert len(list(supplementaries)) == 4 - sftlf_ds = supplementaries['sftlf'] - assert sftlf_ds.facets['mip'] == 'fx' - assert sftlf_ds.facets['exp'] == 'piControl' - - # landseaice - settings = product.settings['mask_landseaice'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - sftgif_ds = supplementaries['sftgif'] - assert sftgif_ds.facets['mip'] == 'fx' - assert sftgif_ds.facets['exp'] == 'piControl' - - # volume statistics - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert 'volcello' in supplementaries - - # area statistics - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - areacello_ds = supplementaries['areacello'] - assert areacello_ds.facets['mip'] == 'fx' - assert areacello_ds.facets['exp'] == 'piControl' + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'sftlf' def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): @@ -2316,731 +2097,32 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): assert dataset.supplementaries == [] -def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, session): - """Test fx variables with given mips.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - ensemble: r2i1p1f1 - mip: Ofx - mask_landseaice: - mask_out: ice - fx_variables: - sftgif: - mip: fx - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert len(list(supplementaries)) == 2 - sftgif_ds = supplementaries['sftgif'] - assert sftgif_ds.facets['mip'] == 'fx' - volcello_ds = supplementaries['volcello'] - assert volcello_ds.facets['ensemble'] == 'r2i1p1f1' - assert volcello_ds.facets['mip'] == 'Ofx' - - -def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, session): - """Test fx variables with invalid mip.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - mip: INVALID - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " - "'areacella' with mip 'INVALID'") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_fx_vars_invalid_mip_for_var_cmip6(tmp_path, patched_datafinder, - session): - """Test fx variables with invalid mip for variable.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - +def test_wrong_project(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: preproc: - area_statistics: + volume_statistics: operator: mean - fx_variables: - areacella: - mip: Lmon - diagnostics: diagnostic_name: variables: - tas: + tos: preprocessor: preproc - project: CMIP6 - mip: Amon + project: CMIP7 + mip: Omon exp: historical start_year: 2000 end_year: 2005 - ensemble: r1i1p1f1 - grid: gn + ensemble: r1i1p1 additional_datasets: - - {dataset: CanESM5} + - {dataset: CanESM2} scripts: null """) - msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " - "'areacella' with mip 'Lmon'") - with pytest.raises(RecipeError) as rec_err_exp: + msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " + "with mip 'Omon'") + with pytest.raises(RecipeError) as wrong_proj: get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, session): - """Test mip tables search for different fx variables.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - areacello: - mask_landsea: - mask_out: sea - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 4 - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert supplementaries['areacella'].facets['mip'] == 'fx' - assert supplementaries['areacello'].facets['mip'] == 'Ofx' - assert supplementaries['sftlf'].facets['mip'] == 'fx' - assert supplementaries['sftof'].facets['mip'] == 'Ofx' - - -def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, session): - """Test mip tables search for list of different fx variables.""" - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: [ - 'areacella', - 'areacello', - ] - mask_landsea: - mask_out: sea - fx_variables: [ - 'sftlf', - 'sftof', - ] - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 4 - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert supplementaries['areacella'].facets['mip'] == 'fx' - assert supplementaries['areacello'].facets['mip'] == 'Ofx' - assert supplementaries['sftlf'].facets['mip'] == 'fx' - assert supplementaries['sftof'].facets['mip'] == 'Ofx' - - -def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Ofx - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['mip'] == 'Ofx' - - -def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Oyr - exp: piControl - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'Oyr' - assert volcello_ds.facets['exp'] == 'piControl' - - -def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - regrid: - target_grid: 1x1 - scheme: linear - extract_volume: - z_min: 0 - z_max: 100 - annual_statistics: - operator: mean - convert_units: - units: K - area_statistics: - operator: mean - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.ancestors) == 0 - assert len(task.products) == 1 - product = task.products.pop() - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert product.attributes['short_name'] == 'tos' - assert dataset.files - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(dataset.supplementaries) == 2 - - -def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, - session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Omon - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['mip'] == 'Omon' - - -def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, - session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Oyr - - diagnostics: - diagnostic_name: - variables: - o2: - preprocessor: preproc - project: CMIP6 - mip: Oyr - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'o2' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'Oyr' - - -def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP5 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'fx' - - -def test_wrong_project(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP7 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " - "with mip 'Omon'") - with pytest.raises(RecipeError) as wrong_proj: - get_recipe(tmp_path, content, session) - assert str(wrong_proj.value) == INITIALIZATION_ERROR_MSG - assert str(wrong_proj.value.failed_tasks[0].message) == msg - - -def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, session): - """Test that error is raised for invalid fx variable.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - wrong_fx_variable: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Preprocessor function 'area_statistics' does not support " - "supplementary variable 'wrong_fx_variable'") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, session): - """Test that error is raised for fx files available in multiple mips.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Requested fx variable 'volcello' for dataset 'CanESM5' of project " - "'CMIP6' is available in more than one CMOR MIP table for 'CMIP6': " - "['Odec', 'Ofx', 'Omon', 'Oyr']") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, - patched_failing_datafinder, - session): - """Test that no error is raised for fx files available in one mip.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - mask_landseaice: - mask_out: ice - fx_variables: - sftgif: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check mask_landseaice - assert 'mask_landseaice' in product.settings - settings = product.settings['mask_landseaice'] - assert len(settings) == 1 - assert settings['mask_out'] == 'ice' - - # Check legacy method of adding supplementary variables - # Due to failing datafinder, only files in LImon are found even though - # sftgif is available in the tables fx, IyrAnt, IyrGre and LImon - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - sftgif_ds = dataset.supplementaries[0] - assert sftgif_ds.facets['short_name'] == 'sftgif' - assert sftgif_ds.facets['mip'] == 'LImon' - assert len(sftgif_ds.files) == 1 + assert str(wrong_proj.value) == msg def test_multimodel_mask(tmp_path, patched_datafinder, session): diff --git a/tests/integration/test_deprecated_config.py b/tests/integration/test_deprecated_config.py index 8bcf246190..71905b53d2 100644 --- a/tests/integration/test_deprecated_config.py +++ b/tests/integration/test_deprecated_config.py @@ -98,65 +98,3 @@ def test_offline_false_deprecation_config(monkeypatch): monkeypatch.setitem(CFG, 'offline', False) assert CFG['offline'] is False assert CFG['search_esgf'] == 'when_missing' - - -def test_use_legacy_supplementaries_default_cfg(): - """Test that option is added for backwards-compatibility.""" - assert CFG['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_user_cfg(): - """Test that option is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - assert cfg['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_default_session(): - """Test that option is added for backwards-compatibility.""" - session = CFG.start_session('my_session') - assert session['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_user_session(): - """Test that option is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - session = cfg.start_session('my_session') - assert session['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_deprecation_session_setitem(): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - session = CFG.start_session('my_session') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session['use_legacy_supplementaries'] = True - assert session['use_legacy_supplementaries'] is True - - -def test_use_legacy_supplementaries_deprecation_session_update(): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - session = CFG.start_session('my_session') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session.update({'use_legacy_supplementaries': False}) - assert session['use_legacy_supplementaries'] is False - - -def test_use_legacy_supplementaries_true_deprecation_config(monkeypatch): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'use_legacy_supplementaries', True) - assert CFG['use_legacy_supplementaries'] is True - - -def test_use_legacy_supplementaries_false_deprecation_config(monkeypatch): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'use_legacy_supplementaries', False) - assert CFG['use_legacy_supplementaries'] is False diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index 498dd606d9..b2fe9cb599 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -546,56 +546,6 @@ def test_get_default_settings(mocker): } -def test_add_legacy_supplementaries_disabled(): - """Test that `_add_legacy_supplementaries` does nothing when disabled.""" - dataset = Dataset() - dataset.session = {'use_legacy_supplementaries': False} - _recipe._add_legacy_supplementary_datasets(dataset, settings={}) - - -def test_enable_legacy_supplementaries_when_used(mocker, session): - """Test that legacy supplementaries are enabled when used in the recipe.""" - recipe = mocker.create_autospec(_recipe.Recipe, instance=True) - recipe.session = session - recipe._preprocessors = { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - 'fx_variables': 'areacella', - } - } - } - session['use_legacy_supplementaries'] = None - _recipe.Recipe._set_use_legacy_supplementaries(recipe) - - assert session['use_legacy_supplementaries'] is True - - -def test_strip_legacy_supplementaries_when_disabled(mocker, session): - """Test that legacy supplementaries are removed when disabled.""" - recipe = mocker.create_autospec(_recipe.Recipe, instance=True) - recipe.session = session - recipe._preprocessors = { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - 'fx_variables': 'areacella', - } - } - } - session['use_legacy_supplementaries'] = False - _recipe.Recipe._set_use_legacy_supplementaries(recipe) - - assert session['use_legacy_supplementaries'] is False - assert recipe._preprocessors == { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - } - } - } - - def test_set_version(mocker): dataset = Dataset(short_name='tas') diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 1a0114e8c9..f96161d4cf 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -452,8 +452,6 @@ def test_from_recipe_with_supplementary(session, tmp_path): def test_from_recipe_with_skip_supplementary(session, tmp_path): - session['use_legacy_supplementaries'] = False - recipe_txt = textwrap.dedent(""" datasets: @@ -501,7 +499,6 @@ def test_from_recipe_with_skip_supplementary(session, tmp_path): def test_from_recipe_with_automatic_supplementary(session, tmp_path, monkeypatch): - session['use_legacy_supplementaries'] = False def _find_files(self): if self.facets['short_name'] == 'areacello': @@ -1165,6 +1162,24 @@ def test_remove_not_found_supplementaries(): assert len(dataset.supplementaries) == 0 +def test_concatenating_historical_and_future_exps(mocker): + mocker.patch.object(Dataset, 'files', True) + dataset = Dataset( + dataset='dataset1', + short_name='tas', + mip='Amon', + frequency='mon', + project='CMIP6', + exp=['historical', 'ssp585'], + ) + dataset.add_supplementary(short_name='areacella', mip='fx', frequency='fx') + dataset._fix_fx_exp() + + assert len(dataset.supplementaries) == 1 + assert dataset.facets['exp'] == ['historical', 'ssp585'] + assert dataset.supplementaries[0].facets['exp'] == 'historical' + + def test_from_recipe_with_glob(tmp_path, session, mocker): recipe_txt = textwrap.dedent(""" From e29649192a8848a5e0c75f34333987963581b7ad Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 16:00:10 +0200 Subject: [PATCH 12/72] Update code coverage orbs (#2206) Co-authored-by: Valeriu Predoi --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d42e6ada13..9765cf6c40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,8 +2,8 @@ version: 2.1 orbs: - coverage-reporter: codacy/coverage-reporter@11.4.1 - codecov: codecov/codecov@1.2.3 + coverage-reporter: codacy/coverage-reporter@13.13.7 + codecov: codecov/codecov@3.2.5 commands: check_changes: @@ -32,7 +32,7 @@ commands: pytest -n 4 --junitxml=test-reports/report.xml esmvaltool version - store_test_results: - path: test-reports/ + path: test-reports/report.xml - store_artifacts: path: /logs - run: @@ -124,6 +124,9 @@ jobs: - /root/.cache/pip - .mypy_cache - .pytest_cache + - run: + name: Install gpg (required by codecov orb) + command: apt update && apt install -y gpg - codecov/upload: when: always file: 'test-reports/coverage.xml' From 69001686bfabe79aa25574ecb95ba88022e1215d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Benke?= <23501924+joergbenke@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:02:58 +0200 Subject: [PATCH 13/72] Restored usage of numpy in `_mask_with_shp` (#2209) --- esmvalcore/preprocessor/_mask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py index b537702014..741d45686f 100644 --- a/esmvalcore/preprocessor/_mask.py +++ b/esmvalcore/preprocessor/_mask.py @@ -281,8 +281,8 @@ def _mask_with_shp(cube, shapefilename, region_indices=None): if region_indices: regions = [regions[idx] for idx in region_indices] - # Create a mask for the data (np->da) - mask = da.zeros(cube.shape, dtype=bool) + # Create a mask for the data + mask = np.zeros(cube.shape, dtype=bool) # Create a set of x,y points from the cube # 1D regular grids From bb66bb3800323ab2b3ef326ba25913f201f1eb9a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 29 Sep 2023 10:39:36 +0200 Subject: [PATCH 14/72] Remove outdated documentation (#2210) --- doc/recipe/preprocessor.rst | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 26c50df715..93bc519fd8 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -463,11 +463,6 @@ This function requires a land or sea area fraction `ancillary variable`_. This supplementary variable, either ``sftlf`` or ``sftof``, should be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. @@ -514,12 +509,6 @@ but if it is not available it will compute a mask based on This supplementary variable, either ``sftlf`` or ``sftof``, can be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - - If the corresponding ancillary variable is not available (which is the case for some models and almost all observational datasets), the preprocessor attempts to mask the data using Natural Earth mask files (that are @@ -551,11 +540,6 @@ This function requires a land ice area fraction `ancillary variable`_. This supplementary variable ``sftgif`` should be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.mask_landseaice`. Glaciated masking @@ -1776,11 +1760,6 @@ The required supplementary variable, either ``areacella`` for atmospheric variables or ``areacello`` for ocean variables, can be attached to the main dataset as described in :ref:`supplementary_variables`. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the user - wishes to input to the function. More details on this are given in :ref:`Fx - variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -1879,12 +1858,6 @@ as described in :ref:`supplementary_variables`. No depth coordinate is required as this is determined by Iris. -.. deprecated:: 2.8.0 - The optional ``fx_variables`` argument specifies the fx variables that the - user wishes to input to the function. - More details on this are given in - :ref:`Fx variables as cell measures or ancillary variables`. - See also :func:`esmvalcore.preprocessor.volume_statistics`. From 837ce35c91a2a14c9a18a73c2d914cf336f21a0a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 14:15:19 +0100 Subject: [PATCH 15/72] [Condalock] Update Linux condalock file (#2212) Co-authored-by: valeriupredoi --- conda-linux-64.lock | 193 ++++++++++++++++++++++---------------------- 1 file changed, 97 insertions(+), 96 deletions(-) diff --git a/conda-linux-64.lock b/conda-linux-64.lock index a5ec825a8a..5517563cfe 100644 --- a/conda-linux-64.lock +++ b/conda-linux-64.lock @@ -25,10 +25,9 @@ https://conda.anaconda.org/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2# https://conda.anaconda.org/conda-forge/linux-64/binutils-2.40-hdd6e379_0.conda#ccc940fddbc3fcd3d79cd4c654c4b5c4 https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.40-hbdbef99_2.conda#adfebae9fdc63a598495dfe3b006973a https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_2.conda#c28003b0be0494f9a7664389146716ff -https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.0-hd590300_0.conda#71b89db63b5b504e7afc8ad901172e1e +https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.3-hd590300_0.conda#434466e97a4174b0c4de114eb7100550 https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2#a1fd65c7ccbf10880423d82bca54eb54 https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.19.1-hd590300_0.conda#e8c18d865be43e2fb3f7a145b6adf1f5 -https://conda.anaconda.org/conda-forge/linux-64/freexl-1.0.6-h166bdaf_1.tar.bz2#897e772a157faf3330d72dd291486f62 https://conda.anaconda.org/conda-forge/linux-64/fribidi-1.0.10-h36c2ea0_0.tar.bz2#ac7bc6a654f8f41b352b38f4051135f8 https://conda.anaconda.org/conda-forge/linux-64/geos-3.12.0-h59595ed_0.conda#3fdf79ef322c8379ae83be491d805369 https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2#14947d8770185e5153fdd04d4673ed37 @@ -42,30 +41,30 @@ https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.1-h166bdaf_0.tar.bz https://conda.anaconda.org/conda-forge/linux-64/lerc-4.0.0-h27087fc_0.tar.bz2#76bbff344f0134279f225174e9064c8f https://conda.anaconda.org/conda-forge/linux-64/libabseil-20230802.1-cxx17_h59595ed_0.conda#2785ddf4cb0e7e743477991d64353947 https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda#0f683578378cddb223e7fd24f785ab2a -https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_0.conda#e805cbec4c29feb22e019245f7e47b6c +https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hd590300_1.conda#aec6c91c7371c26392a06708a73c70e5 https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2#c965a5aa0d5c1c37ffc62dff36e28400 -https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.18-h0b41bf4_0.conda#6aa9c9de5542ecb07fdda9ca626252d8 +https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.19-hd590300_0.conda#1635570038840ee3f9c71d22aa5b8b6d https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2#6f8720dff19e17ce5d48cfe7f3d2f0a3 https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.5.0-hcb278e6_1.conda#6305a3dd2752c76335295da4e581f2fd https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3 https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-13.2.0-ha4646dd_2.conda#78fdab09d9138851dde2b5fe2a11019e https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.17-h166bdaf_0.tar.bz2#b62b52da46c39ee2bc3c162ac7f1804d https://conda.anaconda.org/conda-forge/linux-64/libjpeg-turbo-2.1.5.1-hd590300_1.conda#323e90742f0f48fc22bea908735f55e6 -https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-h7f98852_0.tar.bz2#39b1328babf85c7c3a61636d9cd50206 +https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.0-hd590300_1.conda#854e3e1623b39777140f199c5f9ab952 https://conda.anaconda.org/conda-forge/linux-64/libnuma-2.0.16-h0b41bf4_1.conda#28bfe2cb11357ccc5be21101a6b7ce86 https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-12.3.0-h0f45ef3_2.conda#4655db64eca78a6fcc4fb654fc1f8d57 https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.18-h36c2ea0_1.tar.bz2#c3788462a6fbddafdb413a9f9053e58d https://conda.anaconda.org/conda-forge/linux-64/libtool-2.4.7-h27087fc_0.conda#f204c8ba400ec475452737094fb81d52 https://conda.anaconda.org/conda-forge/linux-64/libutf8proc-2.8.0-h166bdaf_0.tar.bz2#ede4266dc02e875fe1ea77b25dd43747 https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b -https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.1-hd590300_0.conda#82bf6f63eb15ef719b556b63feec3a77 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-base-1.3.2-hd590300_0.conda#30de3fd9b3b602f7473f30e684eeea8c https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#f36c115f1ee199da648e0597ec2047ad https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0 https://conda.anaconda.org/conda-forge/linux-64/lzo-2.10-h516909a_1000.tar.bz2#bb14fcb13341b81d5eb386423b9d2bac https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-hcb278e6_0.conda#681105bccc2a3f7f1a837d47d39c9179 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1 https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.3-hd590300_0.conda#7bb88ce04c8deb9f7d763ae04a1da72f -https://conda.anaconda.org/conda-forge/linux-64/pixman-0.40.0-h36c2ea0_0.tar.bz2#660e72c82f2e75a6b3fe6a6e75c79f19 +https://conda.anaconda.org/conda-forge/linux-64/pixman-0.42.2-h59595ed_0.conda#700edd63ccd5fc66b70b1c028cea9a68 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2#22dad4df6e8630e8dff2428f6f6a7036 https://conda.anaconda.org/conda-forge/linux-64/rdma-core-28.9-h59595ed_1.conda#aeffb7c06b5f65e55e6c637408dc4100 https://conda.anaconda.org/conda-forge/linux-64/re2-2023.03.02-h8c504da_0.conda#206f8fa808748f6e90599c3368a1114e @@ -81,16 +80,16 @@ https://conda.anaconda.org/conda-forge/linux-64/xorg-xproto-7.0.31-h7f98852_1007 https://conda.anaconda.org/conda-forge/linux-64/xxhash-0.8.2-hd590300_0.conda#f08fb5c89edfc4aadee1c81d4cfb1fa1 https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0 https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h7f98852_2.tar.bz2#4cb3ad778ec2d5a7acbdf254eb1c42ae -https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.2-hc309b26_1.conda#72b0dc52522915a4665a55707cc34af0 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h4d4d85c_2.conda#9ca99452635fe03eb5fa937f5ae604b0 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h4d4d85c_1.conda#eba092fc6de212a01de0065f38fe8bbb -https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h4d4d85c_1.conda#30f9df85ce23cd14faa9a4dfa50cca2b +https://conda.anaconda.org/conda-forge/linux-64/aws-c-cal-0.6.2-h09139f6_2.conda#29c3112841eee851f6f5451f6d705782 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-compression-0.2.17-h184a658_3.conda#c62775b5028b5a4eda25037f9af7f5b3 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-sdkutils-0.1.12-h184a658_2.conda#ba06d81b81ec3eaf4ee83cd47f808134 +https://conda.anaconda.org/conda-forge/linux-64/aws-checksums-0.1.17-h184a658_2.conda#10fcdbd02ba7fa0827fb8f7d94f8375b https://conda.anaconda.org/conda-forge/linux-64/expat-2.5.0-hcb278e6_1.conda#8b9b5aca60558d02ddaa09d599e55920 https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-12.3.0-he2b93b0_2.conda#2f4d8677dc7dd87f93e9abfb2ce86808 https://conda.anaconda.org/conda-forge/linux-64/glog-0.6.0-h6f12383_0.tar.bz2#b31f3565cb84435407594e548a2fb7b2 https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h501b40f_6.conda#c3e9338e15d90106f467377017352b97 -https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_0.conda#43017394a280a42b48d11d2a6e169901 -https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_0.conda#8e3e1cb77c4b355a3776bdfb74095bed +https://conda.anaconda.org/conda-forge/linux-64/libbrotlidec-1.1.0-hd590300_1.conda#f07002e225d7a60a694d42a7bf5ff53f +https://conda.anaconda.org/conda-forge/linux-64/libbrotlienc-1.1.0-hd590300_1.conda#5fc11c6020d421960607d821310fcd4d https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2#4d331e44109e3f0e19b4cb8f9b82f3e1 https://conda.anaconda.org/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda#a1cfcc585f0c42bf8d5546bb1dfb668d https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-13.2.0-h69a702a_2.conda#e75a75a6eaf6f318dae2631158c46575 @@ -106,17 +105,17 @@ https://conda.anaconda.org/conda-forge/linux-64/libzip-1.10.1-h2629f0a_3.conda#a https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.40-hc3806b6_0.tar.bz2#69e2c796349cd9b273890bee0febfe1b https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4 https://conda.anaconda.org/conda-forge/linux-64/s2n-1.3.51-h06160fa_0.conda#cd63086544e897be1006fc2d88ed1fe8 -https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.12-h27826a3_0.tar.bz2#5b8c42eb62e9fc961af70bdd6a26e168 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-h2797004_0.conda#513336054f884f95d9fd925748f41ef3 https://conda.anaconda.org/conda-forge/linux-64/ucx-1.14.1-h64cca9d_5.conda#39aa3b356d10d7e5add0c540945a0944 -https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_1.conda#85552d64cb49f12781668779efc738ec +https://conda.anaconda.org/conda-forge/linux-64/udunits2-2.2.28-h40f5838_2.conda#bafaeb2c6cae03d330ac71ef893fa431 https://conda.anaconda.org/conda-forge/linux-64/xorg-libsm-1.2.4-h7391055_0.conda#93ee23f12bc2e684548181256edd2cf6 https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.3.4-h9c3ff4c_1.tar.bz2#21743a8d2ea0c8cfbbf8fe489b0347df https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-hd590300_5.conda#68c34ec6149623be41a1933ab996a209 https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.5-hfc55251_0.conda#04b88013080254850d6c01ed54810589 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.32-h1a03231_3.conda#9e2dd8e0e39417f2de68ac1018cdc809 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-io-0.13.32-h89a0be2_4.conda#00ec8ea1819d8c80d0ed87a2f190ce0b https://conda.anaconda.org/conda-forge/linux-64/blosc-1.21.5-h0f2a231_0.conda#009521b7ed97cca25f8f997f9e745976 https://conda.anaconda.org/conda-forge/linux-64/boost-cpp-1.78.0-h2c5509c_4.conda#417a9d724dc4b651f4a711d3aa3694e3 -https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_0.conda#aeafb07a327e3f14a796bf081ea07472 +https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.1.0-hd590300_1.conda#39f910d205726805a958da408ca194ba https://conda.anaconda.org/conda-forge/linux-64/freetype-2.12.1-h267a509_2.conda#9ae35c3d96db2c94ce0cef86efdfa2cb https://conda.anaconda.org/conda-forge/linux-64/gcc-12.3.0-h8d2909c_2.conda#e2f2f81f367e14ca1f77a870bda2fe59 https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-12.3.0-h76fc315_2.conda#11517e7b5c910c5b5d6985c0c7eb7f50 @@ -127,9 +126,10 @@ https://conda.anaconda.org/conda-forge/linux-64/libarchive-3.7.2-h039dbb9_0.cond https://conda.anaconda.org/conda-forge/linux-64/libglib-2.78.0-hebfc3b9_0.conda#e618003da3547216310088478e475945 https://conda.anaconda.org/conda-forge/linux-64/libgrpc-1.57.0-ha4d0f93_1.conda#56ce4bcc0e1cd0b4c3d7149010410e9a https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.24-pthreads_h413a1c8_0.conda#6e4ef6ca28655124dcde9bd500e44c32 -https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.19.0-h8fd135c_0.conda#d5d149effb0fe13805b68ac2afd242b1 -https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.1-h8b53f26_1.conda#5b09e13d732dda1a2bc9adc711164f4d +https://conda.anaconda.org/conda-forge/linux-64/libthrift-0.19.0-hb90f79a_1.conda#8cdb7d41faa0260875ba92414c487e2d +https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.6.0-h29866fb_1.conda#4e9afd30f4ccb2f98645e51005f82236 https://conda.anaconda.org/conda-forge/linux-64/libxslt-1.1.37-h0054252_1.conda#f27960e8873abb5476e96ef33bdbdccd +https://conda.anaconda.org/conda-forge/linux-64/minizip-4.0.1-h0ab5242_4.conda#98e9a384eb5e53bfa4bfdc8b6f975403 https://conda.anaconda.org/conda-forge/linux-64/nss-3.92-h1d7d5a4_0.conda#22c89a3d87828fe925b310b9cdf0f574 https://conda.anaconda.org/conda-forge/linux-64/orc-1.9.0-h52d3b3c_2.conda#6e1931d3d8512593f606aa08d9bd5192 https://conda.anaconda.org/conda-forge/linux-64/pandoc-3.1.3-h32600fe_0.conda#8287aeb8462e2d4b235eff788e75919d @@ -140,13 +140,13 @@ https://conda.anaconda.org/conda-forge/noarch/alabaster-0.7.13-pyhd8ed1ab_0.cond https://conda.anaconda.org/conda-forge/linux-64/antlr-python-runtime-4.7.2-py311h38be061_1003.tar.bz2#0ab8f8f0cae99343907fe68cda11baea https://conda.anaconda.org/conda-forge/linux-64/atk-1.0-2.38.0-hd4edc92_1.tar.bz2#6c72ec3e660a51736913ef6ea68c454b https://conda.anaconda.org/conda-forge/noarch/attrs-23.1.0-pyh71513ae_1.conda#3edfead7cedd1ab4400a6c588f3e75f8 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.2-h2e3709c_0.conda#749f3bb860c2b5e23f807bedf10fe05b -https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.12-hc865f51_1.conda#dca45458adcf2a29be153d39f885aadb +https://conda.anaconda.org/conda-forge/linux-64/aws-c-event-stream-0.3.2-hd6ebb48_1.conda#ef9692e74f437004ef47a4363552bcb6 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-http-0.7.13-hc690213_1.conda#c912831e92c565598072243266073161 https://conda.anaconda.org/conda-forge/noarch/backcall-0.2.0-pyh9f0ad1d_0.tar.bz2#6006a6d08a3fa99268a2681c7fb55213 https://conda.anaconda.org/conda-forge/noarch/backports-1.0-pyhd8ed1ab_3.conda#54ca2e08b3220c148a1d8329c2678e02 -https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py311h38be061_7.tar.bz2#ec62b3c5b953cb610f5e2b09cd776caf -https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_0.conda#3db48055eab680e43a122e2c7494e7ae -https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_0.conda#b8128d083dbf6abd472b1a3e98b0b83d +https://conda.anaconda.org/conda-forge/linux-64/backports.zoneinfo-0.2.1-py311h38be061_8.conda#5384590f14dfe6ccd02811236afc9f8e +https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda#f27a24d46e3ea7b70a1f98e50c62508f +https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_1.conda#cce9e7c3f1c307f2a5fb08a2922d6164 https://conda.anaconda.org/conda-forge/linux-64/c-compiler-1.6.0-hd590300_0.conda#ea6c792f792bdd7ae6e7e2dee32f0a48 https://conda.anaconda.org/conda-forge/noarch/certifi-2023.7.22-pyhd8ed1ab_0.conda#7f3dbc9179b4dde7da98dfb151d0ad22 https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_0.tar.bz2#ebb5f5f7dc4f1a3780ef7ea7738db08c @@ -156,12 +156,12 @@ https://conda.anaconda.org/conda-forge/noarch/cloudpickle-2.2.1-pyhd8ed1ab_0.con https://conda.anaconda.org/conda-forge/noarch/codespell-2.2.5-pyhd8ed1ab_0.conda#c73551c990f6e7e9c83cdb8bdbafdeb8 https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2#3faab06a954c2a04039983f2c4a50d99 https://conda.anaconda.org/conda-forge/noarch/cycler-0.11.0-pyhd8ed1ab_0.tar.bz2#a50559fad0affdbb33729a68669ca1cb -https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.2-py311hb755f60_0.conda#81d4eacf7eb2d40beee33aa71e8f94ad +https://conda.anaconda.org/conda-forge/linux-64/cython-3.0.2-py311hb755f60_2.conda#35878142ccd98f076942b40ffb3cfbab https://conda.anaconda.org/conda-forge/noarch/decorator-5.1.1-pyhd8ed1ab_0.tar.bz2#43afe5ab04e35e17ba28649471dd7364 https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.7.1-pyhd8ed1ab_0.tar.bz2#961b3a227b437d82ad7054484cfa71b2 https://conda.anaconda.org/conda-forge/noarch/dill-0.3.7-pyhd8ed1ab_0.conda#5e4f3466526c52bc9af2d2353a1460bd https://conda.anaconda.org/conda-forge/noarch/distlib-0.3.7-pyhd8ed1ab_0.conda#12d8aae6994f342618443a8f05c652a0 -https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_1.conda#ff1b48b5b802afe76597197113b8e64d +https://conda.anaconda.org/conda-forge/linux-64/docutils-0.20.1-py311h38be061_2.conda#33f8066e53679dd4be2355fec849bf01 https://conda.anaconda.org/conda-forge/noarch/dodgy-0.2.1-py_0.tar.bz2#62a69d073f7446c90f417b0787122f5b https://conda.anaconda.org/conda-forge/noarch/entrypoints-0.4-pyhd8ed1ab_0.tar.bz2#3cf04868fee0a029769bd41f4b2fbf2d https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.1.3-pyhd8ed1ab_0.conda#e6518222753f519e911e83136d2158d9 @@ -169,8 +169,9 @@ https://conda.anaconda.org/conda-forge/noarch/execnet-2.0.2-pyhd8ed1ab_0.conda#6 https://conda.anaconda.org/conda-forge/noarch/executing-1.2.0-pyhd8ed1ab_0.tar.bz2#4c1bc140e2be5c8ba6e3acab99e25c50 https://conda.anaconda.org/conda-forge/noarch/filelock-3.12.4-pyhd8ed1ab_0.conda#5173d4b8267a0699a43d73231e0b6596 https://conda.anaconda.org/conda-forge/linux-64/fontconfig-2.14.2-h14ed4e7_0.conda#0f69b688f52ff6da70bccb7ff7001d1d -https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.1-pyh1a96a4e_0.conda#d69753ff6ee3c84a6638921dd95db662 -https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h6b639ba_2.conda#ee8220db21db8094998005990418fe5b +https://conda.anaconda.org/conda-forge/linux-64/freexl-2.0.0-h743c826_0.conda#12e6988845706b2cfbc3bc35c9a61a95 +https://conda.anaconda.org/conda-forge/noarch/fsspec-2023.9.2-pyh1a96a4e_0.conda#9d15cd3a0e944594ab528da37dc72ecc +https://conda.anaconda.org/conda-forge/linux-64/gdk-pixbuf-2.42.10-h6c15284_3.conda#06f97c8b69157d91993af0c4f2e16bdc https://conda.anaconda.org/conda-forge/noarch/geographiclib-1.52-pyhd8ed1ab_0.tar.bz2#6880e7100ebae550a33ce26663316d85 https://conda.anaconda.org/conda-forge/linux-64/gfortran-12.3.0-h499e0f7_2.conda#0558a8c44eb7a18e6682bd3a8ae6dcab https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-12.3.0-h7fe76b4_2.conda#3a749210487c0358b6f135a648cbbf60 @@ -182,26 +183,26 @@ https://conda.anaconda.org/conda-forge/noarch/idna-3.4-pyhd8ed1ab_0.tar.bz2#3427 https://conda.anaconda.org/conda-forge/noarch/imagesize-1.4.1-pyhd8ed1ab_0.tar.bz2#7de5386c8fea29e76b303f37dde4c352 https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda#f800d2da156d08e289b14e87e43c1ae5 https://conda.anaconda.org/conda-forge/noarch/itsdangerous-2.1.2-pyhd8ed1ab_0.tar.bz2#3c3de74912f11d2b590184f03c7cd09b -https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py311h9547e67_0.conda#f53903649188b99e6b44c560c69f5b23 -https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h2582759_0.conda#07745544b144855ed4514a4cf0aadd74 -https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-haa2dc70_1.conda#980d8aca0bc23ca73fa8caa3e7c84c28 +https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.5-py311h9547e67_1.conda#2c65bdf442b0d37aad080c8a4e0d452f +https://conda.anaconda.org/conda-forge/linux-64/lazy-object-proxy-1.9.0-py311h459d7ec_1.conda#7cc99d87755a9e64586a6004c5f0f534 +https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.15-h7f713cb_2.conda#9ab79924a3760f85a799f21bc99bd655 https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-18_linux64_openblas.conda#bcddbb497582ece559465b9cd11042e7 https://conda.anaconda.org/conda-forge/linux-64/libcurl-8.3.0-hca28451_0.conda#4ab41bee09a2d2e08de5f09d6f1eef62 https://conda.anaconda.org/conda-forge/linux-64/libkml-1.3.0-h37653c0_1015.tar.bz2#37d3747dd24d604f63d2610910576e63 -https://conda.anaconda.org/conda-forge/linux-64/libpq-15.4-hfc447b1_0.conda#b9ce311e7aba8b5fc3122254f0a6e97e -https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.1-hbf2b3c1_0.conda#4963f3f12db45a576f2b8fbe9a0b8569 +https://conda.anaconda.org/conda-forge/linux-64/libpq-16.0-hfc447b1_1.conda#e4a9a5ba40123477db33e02a78dffb01 +https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.3.2-hdffd6e0_0.conda#a8661c87c873d8c8f90479318ebf0a17 https://conda.anaconda.org/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2#91e27ef3d05cc772ce627e51cff111c4 -https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_0.conda#59a580306d62ef144c9dd592b5120f36 -https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py311h9f220a4_0.conda#b8aad2507303e04037e8d02d8ac54217 -https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py311h459d7ec_0.conda#9904dc4adb5d547cb21e136f98cb24b0 +https://conda.anaconda.org/conda-forge/linux-64/lxml-4.9.3-py311h1a07684_1.conda#aab51e50d994e58efdfa5382139b0468 +https://conda.anaconda.org/conda-forge/linux-64/lz4-4.3.2-py311h38e4bf4_1.conda#f8e0b648d77bbe44d1fe8af8cc56a590 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-2.1.3-py311h459d7ec_1.conda#71120b5155a0c500826cf81536721a15 https://conda.anaconda.org/conda-forge/noarch/mccabe-0.7.0-pyhd8ed1ab_0.tar.bz2#34fc335fc50eef0b5ea708f2b5f54e0c https://conda.anaconda.org/conda-forge/noarch/mistune-3.0.1-pyhd8ed1ab_0.conda#1dad8397c94e4de97a70de552a7dcf49 -https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.5-py311ha3edf6b_0.conda#7415f24f8c44e44152623d93c5015000 +https://conda.anaconda.org/conda-forge/linux-64/msgpack-python-1.0.6-py311h9547e67_0.conda#e826b71bf3dc8c91ee097663e2bcface https://conda.anaconda.org/conda-forge/noarch/munch-4.0.0-pyhd8ed1ab_0.conda#376b32e8f9d3eacbd625f37d39bd507d https://conda.anaconda.org/conda-forge/noarch/munkres-1.1.4-pyh9f0ad1d_0.tar.bz2#2ba8498c1018c1e9c61eb99b973dfe19 https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.0.0-pyha770c72_0.conda#4eccaeba205f0aed9ac3a9ea58568ca3 https://conda.anaconda.org/conda-forge/noarch/networkx-3.1-pyhd8ed1ab_0.conda#254f787d5068bc89f578bf63893ce8b4 -https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda#5ce6a42505c6e9e6151c54c3ec8d68ea +https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-h488ebb8_3.conda#128c25b7fe6a25286a48f3a6a9b5b6f3 https://conda.anaconda.org/conda-forge/noarch/packaging-23.1-pyhd8ed1ab_0.conda#91cda59e66e1e4afe9476f8ef98f5c30 https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.5.0-pyhd8ed1ab_0.tar.bz2#457c2c8c08e54905d6954e79cb5b5db9 https://conda.anaconda.org/conda-forge/noarch/parso-0.8.3-pyhd8ed1ab_0.tar.bz2#17a565a0c3899244e938cdf417e7b094 @@ -209,7 +210,7 @@ https://conda.anaconda.org/conda-forge/noarch/pathspec-0.11.2-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/pickleshare-0.7.5-py_1003.tar.bz2#415f0ebb6198cc2801c73438a9fb5761 https://conda.anaconda.org/conda-forge/noarch/pkgutil-resolve-name-1.3.10-pyhd8ed1ab_1.conda#405678b942f2481cecdb3e010f4925d9 https://conda.anaconda.org/conda-forge/noarch/pluggy-1.3.0-pyhd8ed1ab_0.conda#2390bd10bed1f3fdc7a537fb5a447d8d -https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h2582759_0.conda#a90f8e278c1cd7064b2713e6b7db87e6 +https://conda.anaconda.org/conda-forge/linux-64/psutil-5.9.5-py311h459d7ec_1.conda#490d7fa8675afd1aa6f1b2332d156a45 https://conda.anaconda.org/conda-forge/noarch/ptyprocess-0.7.0-pyhd3deb0d_0.tar.bz2#359eeb6536da0e687af562ed265ec263 https://conda.anaconda.org/conda-forge/noarch/pure_eval-0.2.2-pyhd8ed1ab_0.tar.bz2#6784285c7e55cb7212efabc79e4c2883 https://conda.anaconda.org/conda-forge/noarch/py-1.11.0-pyh6c4a22f_0.tar.bz2#b4613d7e7a493916d867842a6a148054 @@ -222,11 +223,11 @@ https://conda.anaconda.org/conda-forge/noarch/pyshp-2.3.1-pyhd8ed1ab_0.tar.bz2#9 https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2#2a7de29fb590ca14b5243c4c812c8025 https://conda.anaconda.org/conda-forge/noarch/python-fastjsonschema-2.18.0-pyhd8ed1ab_0.conda#3be9466311564f80f8056c0851fc5bb7 https://conda.anaconda.org/conda-forge/noarch/python-tzdata-2023.3-pyhd8ed1ab_0.conda#2590495f608a63625e165915fb4e2e34 -https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.3.0-py311h459d7ec_0.conda#87b306459b81b7a7aaad37222d537a4f +https://conda.anaconda.org/conda-forge/linux-64/python-xxhash-3.3.0-py311h459d7ec_1.conda#7e2181758f84a9c7e776af10fbb2f1a0 https://conda.anaconda.org/conda-forge/noarch/pytz-2023.3.post1-pyhd8ed1ab_0.conda#c93346b446cd08c169d843ae5fc0da97 -https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_0.conda#30eaaf31141e785a445bf1ede6235fe3 -https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h75c88c4_0.conda#af6d43afe0d179ac83b7e0c16b2caaad -https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.3-py311h46250e7_0.conda#da1b2b57ac17853cfeb4197d0595db45 +https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.1-py311h459d7ec_1.conda#52719a74ad130de8fb5d047dc91f247a +https://conda.anaconda.org/conda-forge/linux-64/pyzmq-25.1.1-py311h75c88c4_1.conda#b858421f6a3052950c33aecd44a905cb +https://conda.anaconda.org/conda-forge/linux-64/rpds-py-0.10.3-py311h46250e7_1.conda#7f5b917bca99c5b9d8b4c692e15eb1a3 https://conda.anaconda.org/conda-forge/noarch/semver-3.0.1-pyhd8ed1ab_0.conda#ed90854ae56fb6edae1f13b4663b21b0 https://conda.anaconda.org/conda-forge/noarch/setoptconf-tmp-0.3.1-pyhd8ed1ab_0.tar.bz2#af3e36d4effb85b9b9f93cd1db0963df https://conda.anaconda.org/conda-forge/noarch/setuptools-68.2.2-pyhd8ed1ab_0.conda#fc2166155db840c634a1291a5c35a709 @@ -243,17 +244,17 @@ https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96 https://conda.anaconda.org/conda-forge/noarch/tomlkit-0.12.1-pyha770c72_0.conda#62f5b331c53d73e2f6c4c130b53518a0 https://conda.anaconda.org/conda-forge/noarch/toolz-0.12.0-pyhd8ed1ab_0.tar.bz2#92facfec94bc02d6ccf42e7173831a36 -https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_0.conda#7d9a31416c18704f55946ff7cf8da5dc -https://conda.anaconda.org/conda-forge/noarch/traitlets-5.10.0-pyhd8ed1ab_0.conda#efd3f63a93621367d4fa6e274c511696 -https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.11-pyhd8ed1ab_0.conda#22776dce28e8ba933e5cbcf20b62c583 +https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_1.conda#a700fcb5cedd3e72d0c75d095c7a6eda +https://conda.anaconda.org/conda-forge/noarch/traitlets-5.10.1-pyhd8ed1ab_0.conda#1bbf337ea62a92bd082d429fbdf82b15 +https://conda.anaconda.org/conda-forge/noarch/types-pyyaml-6.0.12.12-pyhd8ed1ab_0.conda#0cb14c80f66937df894d60626dd1921f https://conda.anaconda.org/conda-forge/noarch/types-urllib3-1.26.25.14-pyhd8ed1ab_0.conda#06118f39abab2ab953276a50b2775509 https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.8.0-pyha770c72_0.conda#5b1be40a26d10a06f6d4f1f9e19fa0c7 -https://conda.anaconda.org/conda-forge/linux-64/ujson-5.7.0-py311hcafe171_0.conda#ec3960b6d13bb60aad9c67f42a801720 +https://conda.anaconda.org/conda-forge/linux-64/ujson-5.8.0-py311hb755f60_0.conda#91e67c62c48444e4efc08fb61835abe8 https://conda.anaconda.org/conda-forge/noarch/untokenize-0.1.1-py_0.tar.bz2#1447ead40f2a01733a9c8dfc32988375 https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-pyhd8ed1ab_2.conda#daf5160ff9cde3a468556965329085b9 https://conda.anaconda.org/conda-forge/noarch/webob-1.8.7-pyhd8ed1ab_0.tar.bz2#a8192f3585f341ea66c60c189580ac67 https://conda.anaconda.org/conda-forge/noarch/wheel-0.41.2-pyhd8ed1ab_0.conda#1ccd092478b3e0ee10d7a891adbf8a4f -https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h2582759_0.conda#15565d8602a78c6a994e4d9fcb391920 +https://conda.anaconda.org/conda-forge/linux-64/wrapt-1.15.0-py311h459d7ec_1.conda#f4d770a09066aaa313b5cc22c0f6e9d1 https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda#82b6df12252e6f32402b96dacc656fec https://conda.anaconda.org/conda-forge/linux-64/xorg-libxrender-0.9.11-hd590300_0.conda#ed67c36f215b310412b2af935bf3e530 https://conda.anaconda.org/conda-forge/noarch/xyzservices-2023.7.0-pyhd8ed1ab_0.conda#aacae3c0eaba0204dc6c5497c93c7992 @@ -261,32 +262,32 @@ https://conda.anaconda.org/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_0.conda#cf30 https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda#2e4d6bc0b14e10f895fc6791a7d9b26a https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.4-pyhd8ed1ab_0.conda#46a2e6e3dfa718ce3492018d5a110dd6 https://conda.anaconda.org/conda-forge/noarch/asgiref-3.7.2-pyhd8ed1ab_0.conda#596932155bf88bb6837141550cb721b0 -https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.6-py311h38be061_0.conda#28b1d4a493fb6acd24cc299d77fed871 +https://conda.anaconda.org/conda-forge/linux-64/astroid-2.15.8-py311h38be061_0.conda#46d70fcb74472aab178991f0231ee3c6 https://conda.anaconda.org/conda-forge/noarch/asttokens-2.4.0-pyhd8ed1ab_0.conda#056f04e51dd63337e8d7c425c18c86f1 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.3-he2921ad_3.conda#29f36ec5e9d3c5384e10395f7e189542 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.5-h3a0376c_1.conda#4cfef5eeaa843749252c94324004075e +https://conda.anaconda.org/conda-forge/linux-64/aws-c-auth-0.7.4-hc8144f4_1.conda#81b00630260ff8c9388ee3913465b208 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-mqtt-0.9.6-h32970c0_2.conda#21dd1cb1e73b0ce2ea31e9f9f692e051 https://conda.anaconda.org/conda-forge/noarch/babel-2.12.1-pyhd8ed1ab_1.conda#ac432e732804a81ddcf29c92ead57cde https://conda.anaconda.org/conda-forge/noarch/backports.functools_lru_cache-1.6.5-pyhd8ed1ab_0.conda#6b1b907661838a75d067a22f87996b2e https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.12.2-pyha770c72_0.conda#a362ff7d976217f8fa78c0f1c4f59717 https://conda.anaconda.org/conda-forge/noarch/bleach-6.0.0-pyhd8ed1ab_0.conda#d48b143d01385872a88ef8417e96c30e https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-h0c91306_1017.conda#3db543896d34fc6804ddfb9239dcb125 https://conda.anaconda.org/conda-forge/noarch/cattrs-23.1.2-pyhd8ed1ab_0.conda#e554f60477143949704bf470f66a81e7 -https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py311h409f033_3.conda#9025d0786dbbe4bc91fd8e85502decce +https://conda.anaconda.org/conda-forge/linux-64/cffi-1.16.0-py311hb3a22ac_0.conda#b3469563ac5e808b0cd92810d0697043 https://conda.anaconda.org/conda-forge/linux-64/cfitsio-4.3.0-hbdc6101_0.conda#797554b8b7603011e8677884381fbcc5 https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2#4fd2c6b53934bd7d96d1f3fdaf99b79f https://conda.anaconda.org/conda-forge/noarch/cligj-0.7.2-pyhd8ed1ab_1.tar.bz2#a29b7c141d6b2de4bb67788a5f107734 -https://conda.anaconda.org/conda-forge/linux-64/coverage-7.3.1-py311h459d7ec_0.conda#d23df37f3a595e8ffca99642ab6df3eb +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.3.1-py311h459d7ec_1.conda#bb0e424cb11a7e86700d0bf69e24faec https://conda.anaconda.org/conda-forge/linux-64/cxx-compiler-1.6.0-h00ab1b0_0.conda#364c6ae36c4e36fcbd4d273cf4db78af -https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_0.conda#5c416db47b7816e437eaf0d46e5c3a3d +https://conda.anaconda.org/conda-forge/linux-64/cytoolz-0.12.2-py311h459d7ec_1.conda#afe341dbe834ae76d2c23157ff00e633 https://conda.anaconda.org/conda-forge/noarch/docformatter-1.7.5-pyhd8ed1ab_0.conda#3a941b6083e945aa87e739a9b85c82e9 https://conda.anaconda.org/conda-forge/noarch/fire-0.5.0-pyhd8ed1ab_0.conda#9fd22aae8d2f319e80f68b295ab91d64 -https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.42.1-py311h459d7ec_0.conda#fc327c0ea015db3b6484eabb37d44e60 +https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.43.0-py311h459d7ec_0.conda#6b1558de70fcb3fe6d6bf3294ab5569e https://conda.anaconda.org/conda-forge/linux-64/fortran-compiler-1.6.0-heb67821_0.conda#b65c49dda97ae497abcbdf3a8ba0018f https://conda.anaconda.org/conda-forge/noarch/geopy-2.4.0-pyhd8ed1ab_0.conda#90faaa7eaeba3cc877074c0916efe30c https://conda.anaconda.org/conda-forge/noarch/gitdb-4.0.10-pyhd8ed1ab_0.conda#3706d2f3d7cb5dae600c833345a76132 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.14.2-nompi_h4f84152_100.conda#2de6a9bc8083b49f09b2f6eb28d3ba3c https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-6.8.0-pyha770c72_0.conda#4e9f59a060c3be52bc4ddc46ee9b6946 -https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.0.1-pyhd8ed1ab_0.conda#d978c61aa5fc2c69380d53ad56b5ae86 +https://conda.anaconda.org/conda-forge/noarch/importlib_resources-6.1.0-pyhd8ed1ab_0.conda#48b0d98e0c0ec810d3ccc2a0926c8c0e https://conda.anaconda.org/conda-forge/noarch/isodate-0.6.1-pyhd8ed1ab_0.tar.bz2#4a62c93c1b5c0b920508ae3fd285eaf5 https://conda.anaconda.org/conda-forge/noarch/isort-5.12.0-pyhd8ed1ab_1.conda#07ed3421bad60867234c7a9282ea39d4 https://conda.anaconda.org/conda-forge/noarch/jedi-0.19.0-pyhd8ed1ab_0.conda#1cd7f70057cdffc10977b613fb75425d @@ -294,50 +295,50 @@ https://conda.anaconda.org/conda-forge/noarch/jinja2-3.1.2-pyhd8ed1ab_1.tar.bz2# https://conda.anaconda.org/conda-forge/noarch/jupyterlab_pygments-0.2.2-pyhd8ed1ab_0.tar.bz2#243f63592c8e449f40cd42eb5cf32f40 https://conda.anaconda.org/conda-forge/noarch/latexcodec-2.0.1-pyh9f0ad1d_0.tar.bz2#8d67904973263afd2985ba56aa2d6bb4 https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-18_linux64_openblas.conda#93dd9ab275ad888ed8113953769af78c -https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-h74d50f4_7.conda#3453ac94a99ad9daf17e8a313d274567 +https://conda.anaconda.org/conda-forge/linux-64/libgd-2.3.3-he9388d3_8.conda#f3abc6e6ab60fa404c23094f5a03ec9b https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-h8d7e28b_2.conda#ed3cd026aa12259ce96c0552873705c9 https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-18_linux64_openblas.conda#a1244707531e5b143c420c70573c8ec5 https://conda.anaconda.org/conda-forge/noarch/logilab-common-1.7.3-py_0.tar.bz2#6eafcdf39a7eb90b6d951cfff59e8d3b https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.6-pyhd8ed1ab_0.tar.bz2#b21613793fcc81d944c76c9f2864a7de -https://conda.anaconda.org/conda-forge/linux-64/mypy-1.5.1-py311h459d7ec_0.conda#c99b944502de2ce602b7d666df977a0c +https://conda.anaconda.org/conda-forge/linux-64/mypy-1.5.1-py311h459d7ec_1.conda#f905742ceba3a2ffb62f27970267e297 https://conda.anaconda.org/conda-forge/noarch/nested-lookup-0.2.25-pyhd8ed1ab_1.tar.bz2#2f59daeb14581d41b1e2dda0895933b2 https://conda.anaconda.org/conda-forge/noarch/nodeenv-1.8.0-pyhd8ed1ab_0.conda#2a75b296096adabbabadd5e9782e5fcc -https://conda.anaconda.org/conda-forge/noarch/partd-1.4.0-pyhd8ed1ab_1.conda#6ceb4e000cbe0b56b290180aea8520e8 +https://conda.anaconda.org/conda-forge/noarch/partd-1.4.1-pyhd8ed1ab_0.conda#acf4b7c0bcd5fa3b0e05801c4d2accd6 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.8.0-pyh1a96a4e_2.tar.bz2#330448ce4403cc74990ac07c555942a1 -https://conda.anaconda.org/conda-forge/linux-64/pillow-10.0.0-py311h0b84326_0.conda#4b24acdc1fbbae9da03147e7d2cf8c8a +https://conda.anaconda.org/conda-forge/linux-64/pillow-10.0.1-py311h8aef010_1.conda#4d66ee2081a7cd444ff6f30d95873eef https://conda.anaconda.org/conda-forge/noarch/pip-23.2.1-pyhd8ed1ab_0.conda#e2783aa3f9235225eec92f9081c5b801 -https://conda.anaconda.org/conda-forge/linux-64/postgresql-15.4-h8972f4a_0.conda#bf6169ef6f83cc04d8b2a72cd5c364bc -https://conda.anaconda.org/conda-forge/linux-64/proj-9.2.1-ha643af7_0.conda#e992387307f4403ba0ec07d009032550 +https://conda.anaconda.org/conda-forge/linux-64/postgresql-16.0-h8972f4a_1.conda#6ce1ab5480d3aa4308654971ac5f731b +https://conda.anaconda.org/conda-forge/linux-64/proj-9.3.0-h1d62c97_1.conda#900fd11ac61d4415d515583fcb570207 https://conda.anaconda.org/conda-forge/noarch/pydocstyle-6.3.0-pyhd8ed1ab_0.conda#7e23a61a7fbaedfef6eb0e1ac775c8e5 https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.2-pyhd8ed1ab_0.conda#6dd662ff5ac9a783e5c940ce9f3fe649 https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984 https://conda.anaconda.org/conda-forge/noarch/referencing-0.30.2-pyhd8ed1ab_0.conda#a33161b983172ba6ef69d5fc850650cd https://conda.anaconda.org/conda-forge/noarch/tinycss2-1.2.1-pyhd8ed1ab_0.tar.bz2#7234c9eefff659501cd2fe0d2ede4d48 -https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.3-pyhd8ed1ab_0.conda#d2724da40562babd0322ab353b110225 +https://conda.anaconda.org/conda-forge/noarch/types-requests-2.31.0.6-pyhd8ed1ab_0.conda#69d8b100b4a9e557e33c06b0d3ba4772 https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.8.0-hd8ed1ab_0.conda#384462e63262a527bda564fa2d9126c0 https://conda.anaconda.org/conda-forge/noarch/url-normalize-1.4.3-pyhd8ed1ab_0.tar.bz2#7c4076e494f0efe76705154ac9302ba6 https://conda.anaconda.org/conda-forge/noarch/urllib3-2.0.5-pyhd8ed1ab_0.conda#3bda70bbeb2920f44db5375af2e5fe38 https://conda.anaconda.org/conda-forge/linux-64/xerces-c-3.2.4-hac6953d_3.conda#297e6a75dc1b6a440cd341a85eab8a00 https://conda.anaconda.org/conda-forge/noarch/yamale-4.0.4-pyh6c4a22f_0.tar.bz2#cc9f59f147740d88679bf1bd94dbe588 https://conda.anaconda.org/conda-forge/noarch/yamllint-1.32.0-pyhd8ed1ab_0.conda#6d2425548b0293a225ca4febd80feaa3 -https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.17-h1678ad6_0.conda#e99777ef77b880a59b6caf4405360694 +https://conda.anaconda.org/conda-forge/linux-64/aws-c-s3-0.3.17-hb5e3142_3.conda#f0eeadc3f7fc9a29b7ce416897056826 https://conda.anaconda.org/conda-forge/linux-64/compilers-1.6.0-ha770c72_0.conda#e2259de4640a51a28c21931ae98e4975 https://conda.anaconda.org/conda-forge/linux-64/cryptography-41.0.4-py311h63ff55d_0.conda#2b14cd05541532521196b0d2e0291ecf https://conda.anaconda.org/conda-forge/noarch/django-4.2.5-pyhd8ed1ab_0.conda#5af47510c844ef54896d6b3ea424b82b https://conda.anaconda.org/conda-forge/noarch/flake8-5.0.4-pyhd8ed1ab_0.tar.bz2#8079ea7dec0a917dd0cb6c257f7ea9ea -https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-h22adcc9_11.conda#514167b60f598eaed3f7a60e1dceb9ee -https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.36-pyhd8ed1ab_0.conda#a7c99acc18eea3d70224dd95dadcfd31 +https://conda.anaconda.org/conda-forge/linux-64/geotiff-1.7.1-hee599c5_13.conda#8c55dacddd589be64b2bd6a5d4264be6 +https://conda.anaconda.org/conda-forge/noarch/gitpython-3.1.37-pyhd8ed1ab_0.conda#8b94c329190fa6814f412adf2ab0f0a2 https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.2.1-h3d44ed6_0.conda#98db5f8813f45e2b29766aff0e4a499c https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-6.8.0-hd8ed1ab_0.conda#b279b07ce18058034e5b3606ba103a8b https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.7.1-pyhd8ed1ab_0.conda#7c27ea1bdbe520bb830dcadd59f55cbf -https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-hcd42e92_5.conda#d871720bf750347506062ba23a91662d +https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.2-hcd42e92_1.conda#b04c039f0bd511533a0d8bc8a7b6835e https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.9.2-nompi_h80fb2b6_112.conda#a19fa6cacf80c8a366572853d5890eb4 -https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-h15f6e67_28.conda#bc9758e23157cb8362e60d3de06aa6fb +https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.1.0-h090f1da_0.conda#c4360eaa543bb3bcbb9cd135eb6fb0fc https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.0-py311h64a7726_0.conda#bf16a9f625126e378302f08e7ed67517 https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda#0809187ef9b89a3d94a5c24d13936236 -https://conda.anaconda.org/conda-forge/linux-64/poppler-23.08.0-hd18248d_0.conda#59a093146aa911da2ca056c1197e3e41 +https://conda.anaconda.org/conda-forge/linux-64/poppler-23.08.0-hf2349cb_2.conda#fb75401ae7e2e3f354dff72e9da95cae https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba -https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311ha169711_0.conda#ad4b6e9be79a89959bb6d7d308027ff2 +https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.1-py311h1facc83_2.conda#8298afb85a731b02dac82e02b6e13ae0 https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.1.0-pyhd8ed1ab_0.conda#06eb685a3a0b146347a58dda979485da https://conda.anaconda.org/conda-forge/noarch/pytest-env-1.0.1-pyhd8ed1ab_0.conda#9da651d84c73bac482cae51613a4d4d6 https://conda.anaconda.org/conda-forge/noarch/pytest-metadata-3.0.0-pyhd8ed1ab_1.conda#8bdcc0f401561213821bf67513abeeff @@ -349,49 +350,49 @@ https://conda.anaconda.org/conda-forge/noarch/requests-2.31.0-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/requirements-detector-1.2.2-pyhd8ed1ab_0.conda#6626918380d99292df110f3c91b6e5ec https://conda.anaconda.org/conda-forge/noarch/stack_data-0.6.2-pyhd8ed1ab_0.conda#e7df0fdd404616638df5ece6e69ba7af https://conda.anaconda.org/conda-forge/linux-64/tiledb-2.16.3-h8c794c1_3.conda#7de728789b0aba16018f726dc5ddbec2 -https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h4dd048b_3.tar.bz2#dbfea4376856bf7bd2121e719cf816e5 -https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.6-pyhd8ed1ab_0.conda#078979d33523cb477bd1916ce41aacc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.23.1-hffbee3f_1.conda#e56fcd606b5311dc13c70279b359f13b -https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h4c7f6c3_1.tar.bz2#c7e54004ffd03f8db0a58ab949f2a00b -https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.1-py311h9547e67_0.conda#db5b3b0093d0d4565e5c89578108402e -https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.9.2-pyhd8ed1ab_0.conda#cce7eeb7eda0124af186a5e9ce9b0fca +https://conda.anaconda.org/conda-forge/linux-64/ukkonen-1.0.1-py311h9547e67_4.conda#586da7df03b68640de14dc3e8bcbf76f +https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.2.7-pyhd8ed1ab_0.conda#cdf0c2aad0ddbcaf41b7fd72e2f73d5a +https://conda.anaconda.org/conda-forge/linux-64/aws-crt-cpp-0.23.1-h94c364a_5.conda#0d9257d4ebe9af80677c178f172d3c39 +https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py311h1f0f07a_2.conda#571c0c47e8dbcf03577935ac818b6696 +https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.1.1-py311h9547e67_1.conda#52d3de443952d33c5cee6b24b172ce96 +https://conda.anaconda.org/conda-forge/noarch/dask-core-2023.9.3-pyhd8ed1ab_0.conda#a7155483171dbc27a7385d1c26e779de https://conda.anaconda.org/conda-forge/noarch/flake8-polyfill-1.0.2-py_0.tar.bz2#a53db35e3d07f0af2eccd59c2a00bffe https://conda.anaconda.org/conda-forge/noarch/identify-2.5.29-pyhd8ed1ab_0.conda#5bdbb1cb692649720b60f261b41760cd https://conda.anaconda.org/conda-forge/noarch/jsonschema-4.19.1-pyhd8ed1ab_0.conda#78aff5d2af74e6537c1ca73017f01f4f -https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.3.1-py311h38be061_0.conda#0cf8259b01ede82c76007996f73f89ed -https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.2-h323ed7e_1.conda#b85c17750b1dce1e97d976edf079dd8b +https://conda.anaconda.org/conda-forge/linux-64/jupyter_core-5.3.2-py311h38be061_0.conda#4e4341e940c0dfa1038c1a2d11fd8c3e +https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.7.2-h17082cf_4.conda#aca12eeff11b240c0e0f52185ac92150 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.1-nompi_hacb5139_102.conda#487a1c19dd3eacfd055ad614e9acde87 -https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.1-py311h320fe9a_0.conda#1692362ba82f0556099f0143f7842de3 +https://conda.anaconda.org/conda-forge/linux-64/pandas-2.1.1-py311h320fe9a_1.conda#a4371a95a8ae703a22949af28467b93d https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.14-ha41ecd1_2.conda#1a66c10f6a0da3dbd2f3a68127e7f6a0 -https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyha770c72_3.conda#5936894aade8240c867d292aa0d980c6 +https://conda.anaconda.org/conda-forge/noarch/pooch-1.7.0-pyhd8ed1ab_4.conda#3cdaf7af08850933662b1e228bc6b5bc https://conda.anaconda.org/conda-forge/noarch/prompt-toolkit-3.0.39-pyha770c72_0.conda#a4986c6bb5b0d05a38855b0880a5f425 -https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.5-pyhd8ed1ab_0.conda#30dc94b05de470e3b579d73d64127656 +https://conda.anaconda.org/conda-forge/noarch/pylint-2.17.6-pyhd8ed1ab_0.conda#71d8c641a688ae0fbe6e4ebba44ec405 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.2.0-pyhd8ed1ab_1.conda#34f7d568bf59d18e3fef8c405cbece21 https://conda.anaconda.org/conda-forge/noarch/pytest-html-3.2.0-pyhd8ed1ab_1.tar.bz2#d5c7a941dfbceaab4b172a56d7918eb0 https://conda.anaconda.org/conda-forge/noarch/requests-cache-1.1.0-pyhd8ed1ab_0.conda#57b89064c125bb9d0e533e018c3eb17a -https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.2-py311h64a7726_1.conda#58af16843fc4469770bdbaf45d3a19de -https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311he06c224_2.conda#10a1953d2f74d292b5de093ceea104b2 +https://conda.anaconda.org/conda-forge/linux-64/scipy-1.11.3-py311h64a7726_0.conda#756e8ac1d784f704c0b22559b4bff7b0 +https://conda.anaconda.org/conda-forge/linux-64/shapely-2.0.1-py311he06c224_3.conda#0494ca2b1c365390d014b1295d79e9a3 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.24.4-pyhd8ed1ab_0.conda#c3feaf947264a59a125e8c26e98c3c5a https://conda.anaconda.org/conda-forge/noarch/yapf-0.40.1-pyhd8ed1ab_0.conda#f269942e802d5e148632143d4c37acc9 -https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.156-he6c2984_2.conda#897c30dedccac6d1f73dd52b14e8e70f +https://conda.anaconda.org/conda-forge/linux-64/aws-sdk-cpp-1.11.156-h6600424_3.conda#6caecdec46acbd4743807b4be6efce33 https://conda.anaconda.org/conda-forge/noarch/bokeh-3.2.2-pyhd8ed1ab_0.conda#30488151f591379db656250b3f5fc0c6 https://conda.anaconda.org/conda-forge/linux-64/cf-units-3.2.0-py311h1f0f07a_0.conda#43a71a823583d75308eaf3a06c8f150b -https://conda.anaconda.org/conda-forge/noarch/distributed-2023.9.2-pyhd8ed1ab_0.conda#ddb4fd6105b4005b312625cef210ba67 +https://conda.anaconda.org/conda-forge/noarch/distributed-2023.9.3-pyhd8ed1ab_0.conda#543fafdd7b325bf16199235ee5f20622 https://conda.anaconda.org/conda-forge/linux-64/esmf-8.4.2-nompi_h9e768e6_3.conda#c330e87e698bae8e7381c0315cf25dd0 -https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.2-py311h815a124_1.conda#19c9d9ac47f4ac001967ce16d5831886 +https://conda.anaconda.org/conda-forge/linux-64/gdal-3.7.2-py311h815a124_4.conda#66d14a3095deb8f62f381938b9eea9ad https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#957a0255ab58aaf394a91725d73ab422 https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.3.1-pyhd8ed1ab_0.conda#b7cc0981484fcb6390e6d341e55618b3 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda#620e754f4344f4c27259ff460a2b9c50 -https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.0-py311h54ef318_0.conda#b67672c2f39ef2912a1814e29e42c7ca +https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.8.0-py311h54ef318_1.conda#20d79e2fe53b49b399f3d36977b05abb https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.2-pyhd8ed1ab_0.conda#61ba076de6530d9301a0053b02f093d2 -https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311he8ad708_102.conda#b48083ba918347f30efa94f7dc694919 +https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.4-nompi_py311he8ad708_103.conda#97b45ba4ff4e46a07dd6c60040256538 https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d -https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.4.0-pyha770c72_1.conda#3fb5ba328a77c9fd71197a46e7f2469a +https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.4.0-pyha770c72_2.conda#09cd3006f61e7a7054405f81362e0a5f https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.39-hd8ed1ab_0.conda#4bbbe67d5df19db30f04b8e344dc9976 https://conda.anaconda.org/conda-forge/noarch/pylint-plugin-utils-0.7-pyhd8ed1ab_0.tar.bz2#1657976383aee04dbb3ae3bdf654bb58 -https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_0.conda#3a00b1b08d8c01b1a3bfa686b9152df2 -https://conda.anaconda.org/conda-forge/noarch/xarray-2023.8.0-pyhd8ed1ab_0.conda#a8104cede521616573e228c27f9edc97 +https://conda.anaconda.org/conda-forge/linux-64/python-stratify-0.3.0-py311h1f0f07a_1.conda#cd36a89a048ad2bcc6d8b43f648fb1d0 +https://conda.anaconda.org/conda-forge/noarch/xarray-2023.9.0-pyhd8ed1ab_0.conda#158c89bbc0f2597f33e8ce1aea59e0ee https://conda.anaconda.org/conda-forge/linux-64/cartopy-0.22.0-py311h320fe9a_0.conda#1271b2375735e2aaa6d6770dbe2ad087 https://conda.anaconda.org/conda-forge/noarch/cf_xarray-0.8.4-pyhd8ed1ab_0.conda#18472f8f9452f962fe0bcb1b8134b494 https://conda.anaconda.org/conda-forge/noarch/dask-jobqueue-0.8.2-pyhd8ed1ab_0.conda#cc344a296a41369bcb05f7216661cec8 @@ -399,8 +400,8 @@ https://conda.anaconda.org/conda-forge/noarch/esgf-pyclient-0.3.1-pyh1a96a4e_2.t https://conda.anaconda.org/conda-forge/noarch/esmpy-8.4.2-pyhc1e730c_4.conda#ddcf387719b2e44df0cc4dd467643951 https://conda.anaconda.org/conda-forge/linux-64/fiona-1.9.4-py311hbac4ec9_0.conda#1d3445f5f7fa002a1c149c405376f012 https://conda.anaconda.org/conda-forge/linux-64/graphviz-8.1.0-h28d9a01_0.conda#33628e0e3de7afd2c8172f76439894cb -https://conda.anaconda.org/conda-forge/noarch/ipython-8.15.0-pyh0d859eb_0.conda#6392e665cbdaa780ca2b7a01ac34bb4b -https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-h1935d02_4_cpu.conda#f5efd1ff369209712c6277fd2f3b6c03 +https://conda.anaconda.org/conda-forge/noarch/ipython-8.16.0-pyh0d859eb_0.conda#30450c4d405002b8f4aa4322cd70184d +https://conda.anaconda.org/conda-forge/linux-64/libarrow-13.0.0-h1935d02_5_cpu.conda#105be62a1a03a1db24485923ffa8e07e https://conda.anaconda.org/conda-forge/noarch/nbclient-0.8.0-pyhd8ed1ab_0.conda#e78da91cf428faaf05701ce8cc8f2f9b https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/noarch/pylint-celery-0.3-py_1.tar.bz2#e29456a611a62d3f26105a2f9c68f759 @@ -410,9 +411,9 @@ https://conda.anaconda.org/conda-forge/noarch/iris-3.7.0-pyha770c72_0.conda#dccc https://conda.anaconda.org/conda-forge/noarch/nbconvert-core-7.8.0-pyhd8ed1ab_0.conda#62345c9e24f898bf492979be84a6eb0a https://conda.anaconda.org/conda-forge/noarch/prospector-1.10.2-pyhd8ed1ab_0.conda#2c536985982f7e531df8d640f554008a https://conda.anaconda.org/conda-forge/noarch/py-cordex-0.6.3-pyhd8ed1ab_0.conda#de9f9392273a1e6095930a21561a37e9 -https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_4_cpu.conda#cf1ad9f013404a12bd0c823ca9b99742 -https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_3.tar.bz2#64a77de29fde80aef5013ddf5e62a564 -https://conda.anaconda.org/conda-forge/noarch/dask-2023.9.2-pyhd8ed1ab_0.conda#29e33df59c9eac1a599b9cd18d54b4d3 +https://conda.anaconda.org/conda-forge/linux-64/pyarrow-13.0.0-py311h39c9aba_5_cpu.conda#9d4dc425cdfaee0ec68a4b8d0bc07909 +https://conda.anaconda.org/conda-forge/linux-64/pydot-1.4.2-py311h38be061_4.conda#5c223cb0d9c05552bf9d1586a92720b2 +https://conda.anaconda.org/conda-forge/noarch/dask-2023.9.3-pyhd8ed1ab_0.conda#5b32dc4eb1f5e3097cc33fb0e331b3a4 https://conda.anaconda.org/conda-forge/noarch/nbconvert-pandoc-7.8.0-pyhd8ed1ab_0.conda#1dba1a577df2625a24667612a069e91c https://conda.anaconda.org/conda-forge/noarch/prov-2.0.0-pyhd3deb0d_0.tar.bz2#aa9b3ad140f6c0668c646f32e20ccf82 https://conda.anaconda.org/conda-forge/noarch/iris-esmf-regrid-0.8.0-pyhd8ed1ab_0.conda#56e85460d22fa7d4fb06300f785dd1e1 From bd3b9a4638fb35effee527b708d189d13b476ec2 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 4 Oct 2023 16:50:32 +0200 Subject: [PATCH 16/72] Remove deprecated preprocessor function `cleanup` (#2215) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- esmvalcore/preprocessor/__init__.py | 7 --- esmvalcore/preprocessor/_io.py | 46 ------------------ .../preprocessor/_io/test_cleanup.py | 48 ------------------- .../preprocessor/test_preprocessor_file.py | 20 +------- 4 files changed, 1 insertion(+), 120 deletions(-) delete mode 100644 tests/integration/preprocessor/_io/test_cleanup.py diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index d66592a5b9..4a5027bce5 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -29,7 +29,6 @@ from ._io import ( _get_debug_filename, _sort_products, - cleanup, concatenate, load, save, @@ -188,7 +187,6 @@ 'remove_supplementary_variables', # Save to file 'save', - 'cleanup', ] TIME_PREPROCESSORS = [ @@ -491,11 +489,6 @@ def save(self): 'save', input_files=self._input_files, **self.settings['save']) - if 'cleanup' in self.settings: - preprocess([], - 'cleanup', - input_files=self._input_files, - **self.settings['cleanup']) def close(self): """Close the file.""" diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 596b3b331a..4da6736501 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -4,8 +4,6 @@ import copy import logging import os -import shutil -import warnings from itertools import groupby from pathlib import Path from typing import Optional @@ -18,7 +16,6 @@ from cf_units import suppress_errors from iris.cube import CubeList -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import merge_cube_attributes from .._task import write_ncl_settings @@ -143,7 +140,6 @@ def load( ------ ValueError Cubes are empty. - """ file = Path(file) logger.debug("Loading:\n%s", file) @@ -349,48 +345,6 @@ def _get_debug_filename(filename, step): return filename -def cleanup(files, remove=None): - """Clean up after running the preprocessor. - - Warning - ------- - .. deprecated:: 2.8.0 - This function is no longer used and has been deprecated since - ESMValCore version 2.8.0. It is scheduled for removal in version - 2.10.0. - - Parameters - ---------- - files: list of Path - Preprocessor output files (will not be removed if not in `removed`). - remove: list of Path or None, optional (default: None) - Files or directories to remove. - - Returns - ------- - list of Path - Preprocessor output files. - - """ - deprecation_msg = ( - "The preprocessor function `cleanup` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in version " - "2.10.0." - ) - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) - - if remove is None: - remove = [] - - for path in remove: - if os.path.isdir(path): - shutil.rmtree(path) - elif os.path.isfile(path): - os.remove(path) - - return files - - def _sort_products(products): """Sort preprocessor output files by their order in the recipe.""" return sorted( diff --git a/tests/integration/preprocessor/_io/test_cleanup.py b/tests/integration/preprocessor/_io/test_cleanup.py deleted file mode 100644 index 3ef98b8574..0000000000 --- a/tests/integration/preprocessor/_io/test_cleanup.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Integration tests for :func:`esmvalcore.preprocessor._io.cleanup`""" - -import os -import tempfile -import unittest - -import pytest - -from esmvalcore.exceptions import ESMValCoreDeprecationWarning -from esmvalcore.preprocessor import _io - - -class TestCleanup(unittest.TestCase): - """Tests for :func:`esmvalcore.preprocessor._io.cleanup`""" - - def setUp(self): - """Prepare tests.""" - self.temp_paths = [] - descriptor, temp_file = tempfile.mkstemp('.nc') - os.close(descriptor) - self.temp_paths.append(temp_file) - self.temp_paths.append(tempfile.mkdtemp()) - - def tearDown(self): - for path in self.temp_paths: - if os.path.isfile(path): - os.remove(path) - elif os.path.isdir(path): - os.rmdir(path) - - def test_cleanup(self): - """Test cleanup""" - _io.cleanup([], self.temp_paths) - for path in self.temp_paths: - self.assertFalse(os.path.exists(path)) - - def test_cleanup_when_files_removed(self): - """Test cleanup works even with missing files or folders""" - self.tearDown() - _io.cleanup([], self.temp_paths) - for path in self.temp_paths: - self.assertFalse(os.path.exists(path)) - - def test_deprecation(self): - """Test that deprecation warning is properly raised.""" - msg = "cleanup" - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - _io.cleanup([], []) diff --git a/tests/unit/preprocessor/test_preprocessor_file.py b/tests/unit/preprocessor/test_preprocessor_file.py index 3ebb3385d6..30e4c943cd 100644 --- a/tests/unit/preprocessor/test_preprocessor_file.py +++ b/tests/unit/preprocessor/test_preprocessor_file.py @@ -148,7 +148,7 @@ def test_close(): @mock.patch('esmvalcore.preprocessor.preprocess', autospec=True) -def test_save_no_cleanup(mock_preprocess): +def test_save(mock_preprocess): """Test ``save``.""" product = mock.create_autospec(PreprocessorFile, instance=True) product.settings = {'save': {}} @@ -162,21 +162,3 @@ def test_save_no_cleanup(mock_preprocess): mock.sentinel.cubes, 'save', input_files=mock.sentinel.input_files ), ] - - -@mock.patch('esmvalcore.preprocessor.preprocess', autospec=True) -def test_save_cleanup(mock_preprocess): - """Test ``save``.""" - product = mock.create_autospec(PreprocessorFile, instance=True) - product.settings = {'save': {}, 'cleanup': {}} - product._cubes = mock.sentinel.cubes - product._input_files = mock.sentinel.input_files - - PreprocessorFile.save(product) - - assert mock_preprocess.mock_calls == [ - mock.call( - mock.sentinel.cubes, 'save', input_files=mock.sentinel.input_files - ), - mock.call([], 'cleanup', input_files=mock.sentinel.input_files), - ] From 49d96354740cf02b32646bc55e71647437a205e9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:52:54 +0200 Subject: [PATCH 17/72] Removed deprecated configuration option `offline` (#2213) --- esmvalcore/_main.py | 12 ---- esmvalcore/config/_config_validators.py | 50 ++------------ tests/integration/test_deprecated_config.py | 76 --------------------- tests/integration/test_main.py | 6 -- 4 files changed, 7 insertions(+), 137 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index da5cdd0262..760ec054bd 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -337,7 +337,6 @@ def run(self, max_datasets=None, max_years=None, skip_nonexistent=None, - offline=None, search_esgf=None, diagnostics=None, check_level=None, @@ -365,15 +364,6 @@ def run(self, Maximum number of years to use. skip_nonexistent: bool, optional If True, the run will not fail if some datasets are not available. - offline: bool, optional - If True, the tool will not download missing data from ESGF. - - .. deprecated:: 2.8.0 - This option has been deprecated in ESMValCore version 2.8.0 and - is scheduled for removal in version 2.10.0. Please use the - options `search_esgf=never` (for `offline=True`) or - `search_esgf=when_missing` (for `offline=False`). These are - exact replacements. search_esgf: str, optional If `never`, disable automatic download of data from the ESGF. If `when_missing`, enable the automatic download of files that are not @@ -405,8 +395,6 @@ def run(self, session['max_datasets'] = max_datasets if max_years is not None: session['max_years'] = max_years - if offline is not None: - session['offline'] = offline if search_esgf is not None: session['search_esgf'] = search_esgf if skip_nonexistent is not None: diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 8f5e47375a..867370ae77 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -7,7 +7,7 @@ from collections.abc import Callable, Iterable from functools import lru_cache, partial from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import Any, Optional, Union from packaging import version @@ -23,9 +23,6 @@ InvalidConfigParameter, ) -if TYPE_CHECKING: - from ._validated_config import ValidatedConfig - logger = logging.getLogger(__name__) @@ -288,7 +285,6 @@ def validate_diagnostics( 'extra_facets_dir': validate_pathtuple, 'log_level': validate_string, 'max_parallel_tasks': validate_int_or_none, - 'offline': validate_bool, 'output_dir': validate_path, 'output_file_type': validate_string, 'profile_diagnostic': validate_bool, @@ -339,44 +335,12 @@ def _handle_deprecation( warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) -def deprecate_offline( - validated_config: ValidatedConfig, - value: Any, - validated_value: Any, -) -> None: - """Deprecate ``offline`` option. - - Parameters - ---------- - validated_config: ValidatedConfig - ``ValidatedConfig`` instance which will be modified in place. - value: Any - Raw input value for ``offline`` option. - validated_value: Any - Validated value for ``offline`` option. - - """ - option = 'offline' - deprecated_version = '2.8.0' - remove_version = '2.10.0' - more_info = ( - " Please use the options `search_esgf=never` (for `offline=True`) or " - "`search_esgf=when_missing` (for `offline=False`) instead. These are " - "exact replacements." - ) - _handle_deprecation(option, deprecated_version, remove_version, more_info) - if validated_value: - validated_config['search_esgf'] = 'never' - else: - validated_config['search_esgf'] = 'when_missing' - - -_deprecators: dict[str, Callable] = { - 'offline': deprecate_offline, -} +# Example usage: see removed files in +# https://github.com/ESMValGroup/ESMValCore/pull/2213 +_deprecators: dict[str, Callable] = {} # Default values for deprecated options -_deprecated_options_defaults: dict[str, Any] = { - 'offline': True, -} +# Example usage: see removed files in +# https://github.com/ESMValGroup/ESMValCore/pull/2213 +_deprecated_options_defaults: dict[str, Any] = {} diff --git a/tests/integration/test_deprecated_config.py b/tests/integration/test_deprecated_config.py index 71905b53d2..8dec085134 100644 --- a/tests/integration/test_deprecated_config.py +++ b/tests/integration/test_deprecated_config.py @@ -1,8 +1,6 @@ import warnings from pathlib import Path -import pytest - import esmvalcore from esmvalcore.config import CFG, Config from esmvalcore.exceptions import ESMValCoreDeprecationWarning @@ -24,77 +22,3 @@ def test_no_deprecation_user_cfg(): cfg = Config(CFG.copy()) cfg.load_from_file(config_file) cfg.start_session('my_session') - - -def test_offline_default_cfg(): - """Test that ``offline`` is added for backwards-compatibility.""" - assert CFG['search_esgf'] == 'never' - assert CFG['offline'] is True - - -def test_offline_user_cfg(): - """Test that ``offline`` is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - assert cfg['search_esgf'] == 'never' - assert cfg['offline'] is True - - -def test_offline_default_session(): - """Test that ``offline`` is added for backwards-compatibility.""" - session = CFG.start_session('my_session') - assert session['search_esgf'] == 'never' - assert session['offline'] is True - - -def test_offline_user_session(): - """Test that ``offline`` is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - session = cfg.start_session('my_session') - assert session['search_esgf'] == 'never' - assert session['offline'] is True - - -def test_offline_deprecation_session_setitem(): - """Test that the usage of offline is deprecated.""" - msg = "offline" - session = CFG.start_session('my_session') - session.pop('search_esgf') # test automatic addition of search_esgf - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session['offline'] = True - assert session['offline'] is True - assert session['search_esgf'] == 'never' - - -def test_offline_deprecation_session_update(): - """Test that the usage of offline is deprecated.""" - msg = "offline" - session = CFG.start_session('my_session') - session.pop('search_esgf') # test automatic addition of search_esgf - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session.update({'offline': False}) - assert session['offline'] is False - assert session['search_esgf'] == 'when_missing' - - -def test_offline_true_deprecation_config(monkeypatch): - """Test that the usage of offline is deprecated.""" - msg = "offline" - monkeypatch.delitem(CFG, 'search_esgf') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'offline', True) - assert CFG['offline'] is True - assert CFG['search_esgf'] == 'never' - - -def test_offline_false_deprecation_config(monkeypatch): - """Test that the usage of offline is deprecated.""" - msg = "offline" - monkeypatch.delitem(CFG, 'search_esgf') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'offline', False) - assert CFG['offline'] is False - assert CFG['search_esgf'] == 'when_missing' diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 3ea26c56ed..4eb578150d 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -119,12 +119,6 @@ def test_run_with_max_datasets(): run() -@patch('esmvalcore._main.ESMValTool.run', new=wrapper(ESMValTool.run)) -def test_run_with_offline(): - with arguments('esmvaltool', 'run', 'recipe.yml', '--offline'): - run() - - @patch('esmvalcore._main.ESMValTool.run', new=wrapper(ESMValTool.run)) def test_run_with_search_esgf(): with arguments('esmvaltool', 'run', 'recipe.yml', '--search_esgf=always'): From 13a444eebd0806e5b7633ca12b3ca1dab7bf5668 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:51:24 +0200 Subject: [PATCH 18/72] Added `get_time_bounds` and `get_next_month` to public API (#2214) --- esmvalcore/cmor/_fixes/cmip6/iitm_esm.py | 2 +- esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py | 2 +- esmvalcore/cmor/_fixes/fix.py | 85 +----------------- esmvalcore/cmor/_fixes/shared.py | 93 ++++++++++++++++++++ esmvalcore/cmor/fixes.py | 9 +- esmvalcore/preprocessor/_time.py | 2 +- tests/integration/cmor/_fixes/test_shared.py | 38 ++++++++ tests/unit/cmor/test_fixes.py | 2 + tests/unit/cmor/test_generic_fix.py | 39 +------- 9 files changed, 146 insertions(+), 126 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py index a6dee710c9..6ed2108ff7 100644 --- a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py +++ b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor._fixes.fix import get_time_bounds +from esmvalcore.cmor.fixes import get_time_bounds from ..common import OceanFixGrid from ..fix import Fix diff --git a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py index 5e27377b0d..e4c2cc420e 100644 --- a/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py +++ b/esmvalcore/cmor/_fixes/cmip6/kace_1_0_g.py @@ -3,7 +3,7 @@ import numpy as np -from esmvalcore.cmor._fixes.fix import get_time_bounds +from esmvalcore.cmor.fixes import get_time_bounds from ..common import ClFixHybridHeightCoord, OceanFixGrid from ..fix import Fix diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 4b5163d4a8..a6156a231c 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -6,7 +6,6 @@ import logging import tempfile from collections.abc import Sequence -from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -25,8 +24,8 @@ _get_single_cube, _is_unstructured_grid, ) +from esmvalcore.cmor.fixes import get_time_bounds from esmvalcore.cmor.table import get_var_info -from esmvalcore.iris_helpers import date2num if TYPE_CHECKING: from esmvalcore.cmor.table import CoordinateInfo, VariableInfo @@ -325,88 +324,6 @@ def get_fixed_filepath( return output_dir / Path(filepath).name -def get_next_month(month: int, year: int) -> tuple[int, int]: - """Get next month and year. - - Parameters - ---------- - month: - Current month. - year: - Current year. - - Returns - ------- - tuple[int, int] - Next month and next year. - - """ - if month != 12: - return month + 1, year - return 1, year + 1 - - -def get_time_bounds(time: Coord, freq: str): - """Get bounds for time coordinate. - - Parameters - ---------- - time: - Time coordinate. - freq: - Frequency. - - Returns - ------- - np.ndarray - Time bounds - - Raises - ------ - NotImplementedError - Non-supported frequency is given. - - """ - bounds = [] - dates = time.units.num2date(time.points) - for step, date in enumerate(dates): - month = date.month - year = date.year - if freq in ['mon', 'mo']: - next_month, next_year = get_next_month(month, year) - min_bound = date2num(datetime(year, month, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), - time.units, time.dtype) - elif freq == 'yr': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), - time.units, time.dtype) - elif freq == 'dec': - min_bound = date2num(datetime(year, 1, 1, 0, 0), - time.units, time.dtype) - max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), - time.units, time.dtype) - else: - delta = { - 'day': 12.0 / 24, - '6hr': 3.0 / 24, - '3hr': 1.5 / 24, - '1hr': 0.5 / 24, - } - if freq not in delta: - raise NotImplementedError( - f"Cannot guess time bounds for frequency '{freq}'" - ) - point = time.points[step] - min_bound = point - delta[freq] - max_bound = point + delta[freq] - bounds.append([min_bound, max_bound]) - - return np.array(bounds) - - class GenericFix(Fix): """Class providing generic fixes for all datasets.""" diff --git a/esmvalcore/cmor/_fixes/shared.py b/esmvalcore/cmor/_fixes/shared.py index 0e4dfead08..55ccc8697f 100644 --- a/esmvalcore/cmor/_fixes/shared.py +++ b/esmvalcore/cmor/_fixes/shared.py @@ -1,6 +1,7 @@ """Shared functions for fixes.""" import logging import os +from datetime import datetime from functools import lru_cache import dask.array as da @@ -9,8 +10,11 @@ import pandas as pd from cf_units import Unit from iris import NameConstraint +from iris.coords import Coord from scipy.interpolate import interp1d +from esmvalcore.iris_helpers import date2num + logger = logging.getLogger(__name__) @@ -411,3 +415,92 @@ def fix_ocean_depth_coord(cube): depth_coord.units = 'm' depth_coord.long_name = 'ocean depth coordinate' depth_coord.attributes = {'positive': 'down'} + + +def get_next_month(month: int, year: int) -> tuple[int, int]: + """Get next month and year. + + Parameters + ---------- + month: + Current month. + year: + Current year. + + Returns + ------- + tuple[int, int] + Next month and next year. + + """ + if month != 12: + return month + 1, year + return 1, year + 1 + + +def get_time_bounds(time: Coord, freq: str) -> np.ndarray: + """Get bounds for time coordinate. + + For monthly data, use the first day of the current month and the first day + of the next month. For yearly or decadal data, use 1 January of the current + year and 1 January of the next year or 10 years from the current year. For + other frequencies (daily, 6-hourly, 3-hourly, hourly), half of the + frequency is subtracted/added from the current point in time to get the + bounds. + + Parameters + ---------- + time: + Time coordinate. + freq: + Frequency. + + Returns + ------- + np.ndarray + Time bounds + + Raises + ------ + NotImplementedError + Non-supported frequency is given. + + """ + bounds = [] + dates = time.units.num2date(time.points) + for step, date in enumerate(dates): + month = date.month + year = date.year + if freq in ['mon', 'mo']: + next_month, next_year = get_next_month(month, year) + min_bound = date2num(datetime(year, month, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(next_year, next_month, 1, 0, 0), + time.units, time.dtype) + elif freq == 'yr': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 1, 1, 1, 0, 0), + time.units, time.dtype) + elif freq == 'dec': + min_bound = date2num(datetime(year, 1, 1, 0, 0), + time.units, time.dtype) + max_bound = date2num(datetime(year + 10, 1, 1, 0, 0), + time.units, time.dtype) + else: + delta = { + 'day': 12.0 / 24, + '6hr': 3.0 / 24, + '3hr': 1.5 / 24, + '1hr': 0.5 / 24, + } + if freq not in delta: + raise NotImplementedError( + f"Cannot guess time bounds for frequency '{freq}'" + ) + point = time.points[step] + min_bound = point - delta[freq] + max_bound = point + delta[freq] + bounds.append([min_bound, max_bound]) + + return np.array(bounds) diff --git a/esmvalcore/cmor/fixes.py b/esmvalcore/cmor/fixes.py index e5931b0f0f..534aa3bd94 100644 --- a/esmvalcore/cmor/fixes.py +++ b/esmvalcore/cmor/fixes.py @@ -1,8 +1,15 @@ """Functions for fixing specific issues with datasets.""" -from ._fixes.shared import add_altitude_from_plev, add_plev_from_altitude +from ._fixes.shared import ( + add_altitude_from_plev, + add_plev_from_altitude, + get_next_month, + get_time_bounds, +) __all__ = [ 'add_altitude_from_plev', 'add_plev_from_altitude', + 'get_time_bounds', + 'get_next_month', ] diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 6c363379e5..f39566e51f 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -23,7 +23,7 @@ from iris.cube import Cube, CubeList from iris.time import PartialDateTime -from esmvalcore.cmor._fixes.fix import get_next_month, get_time_bounds +from esmvalcore.cmor.fixes import get_next_month, get_time_bounds from esmvalcore.iris_helpers import date2num from ._shared import get_iris_analysis_operation, operator_accept_weights diff --git a/tests/integration/cmor/_fixes/test_shared.py b/tests/integration/cmor/_fixes/test_shared.py index 92d2cfba2a..114dec7e0a 100644 --- a/tests/integration/cmor/_fixes/test_shared.py +++ b/tests/integration/cmor/_fixes/test_shared.py @@ -7,6 +7,7 @@ import pytest from cf_units import Unit from iris import NameConstraint +from iris.coords import AuxCoord from esmvalcore.cmor._fixes.shared import ( _map_on_filled, @@ -25,6 +26,7 @@ get_altitude_to_pressure_func, get_bounds_cube, get_pressure_to_altitude_func, + get_time_bounds, round_coordinates, ) @@ -620,3 +622,39 @@ def test_fix_ocean_depth_coord(): assert depth_coord.units == 'm' assert depth_coord.long_name == 'ocean depth coordinate' assert depth_coord.attributes == {'positive': 'down'} + + +@pytest.fixture +def time_coord(): + """Time coordinate.""" + time_coord = AuxCoord( + [15, 350], + standard_name='time', + units='days since 1850-01-01' + ) + return time_coord + + +@pytest.mark.parametrize( + 'freq,expected_bounds', + [ + ('mon', [[0, 31], [334, 365]]), + ('mo', [[0, 31], [334, 365]]), + ('yr', [[0, 365], [0, 365]]), + ('dec', [[0, 3652], [0, 3652]]), + ('day', [[14.5, 15.5], [349.5, 350.5]]), + ('6hr', [[14.875, 15.125], [349.875, 350.125]]), + ('3hr', [[14.9375, 15.0625], [349.9375, 350.0625]]), + ('1hr', [[14.97916666, 15.020833333], [349.97916666, 350.020833333]]), + ] +) +def test_get_time_bounds(time_coord, freq, expected_bounds): + """Test ``get_time_bounds`.""" + bounds = get_time_bounds(time_coord, freq) + np.testing.assert_allclose(bounds, expected_bounds) + + +def test_get_time_bounds_invalid_freq_fail(time_coord): + """Test ``get_time_bounds`.""" + with pytest.raises(NotImplementedError): + get_time_bounds(time_coord, 'invalid_freq') diff --git a/tests/unit/cmor/test_fixes.py b/tests/unit/cmor/test_fixes.py index 24e4acde52..c1a87e1bc1 100644 --- a/tests/unit/cmor/test_fixes.py +++ b/tests/unit/cmor/test_fixes.py @@ -8,6 +8,8 @@ @pytest.mark.parametrize('func', [ 'add_altitude_from_plev', 'add_plev_from_altitude', + 'get_next_month', + 'get_time_bounds', ]) def test_imports(func): assert func in fixes.__all__ diff --git a/tests/unit/cmor/test_generic_fix.py b/tests/unit/cmor/test_generic_fix.py index 54703b93f5..4294fe0752 100644 --- a/tests/unit/cmor/test_generic_fix.py +++ b/tests/unit/cmor/test_generic_fix.py @@ -2,26 +2,14 @@ from unittest.mock import sentinel -import numpy as np import pytest from iris.coords import AuxCoord from iris.cube import Cube, CubeList -from esmvalcore.cmor._fixes.fix import GenericFix, get_time_bounds +from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor.table import get_var_info -@pytest.fixture -def time_coord(): - """Time coordinate.""" - time_coord = AuxCoord( - [15, 350], - standard_name='time', - units='days since 1850-01-01' - ) - return time_coord - - @pytest.fixture def generic_fix(): """Generic fix object.""" @@ -30,31 +18,6 @@ def generic_fix(): return GenericFix(vardef, extra_facets=extra_facets) -@pytest.mark.parametrize( - 'freq,expected_bounds', - [ - ('mon', [[0, 31], [334, 365]]), - ('mo', [[0, 31], [334, 365]]), - ('yr', [[0, 365], [0, 365]]), - ('dec', [[0, 3652], [0, 3652]]), - ('day', [[14.5, 15.5], [349.5, 350.5]]), - ('6hr', [[14.875, 15.125], [349.875, 350.125]]), - ('3hr', [[14.9375, 15.0625], [349.9375, 350.0625]]), - ('1hr', [[14.97916666, 15.020833333], [349.97916666, 350.020833333]]), - ] -) -def test_get_time_bounds(time_coord, freq, expected_bounds): - """Test ``get_time_bounds`.""" - bounds = get_time_bounds(time_coord, freq) - np.testing.assert_allclose(bounds, expected_bounds) - - -def test_get_time_bounds_invalid_freq_fail(time_coord): - """Test ``get_time_bounds`.""" - with pytest.raises(NotImplementedError): - get_time_bounds(time_coord, 'invalid_freq') - - def test_generic_fix_empty_long_name(generic_fix, monkeypatch): """Test ``GenericFix``.""" # Artificially set long_name to empty string for test From 9b323aae15f53b47b72ad02332e618802d873b24 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:47:16 +0200 Subject: [PATCH 19/72] Cleaned and extended function that extracts datetimes from paths (#2181) Co-authored-by: Valeriu Predoi --- esmvalcore/dataset.py | 2 +- esmvalcore/esgf/_search.py | 2 +- esmvalcore/local.py | 162 ++++++++++++++++------------------ tests/unit/local/test_time.py | 93 ++++++++++++++++--- 4 files changed, 155 insertions(+), 104 deletions(-) diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 849a9a579e..2b2230a59c 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -857,7 +857,7 @@ def _update_timerange(self): dataset.facets.pop('timerange') dataset.supplementaries = [] check.data_availability(dataset) - intervals = [_get_start_end_date(f.name) for f in dataset.files] + intervals = [_get_start_end_date(f) for f in dataset.files] min_date = min(interval[0] for interval in intervals) max_date = max(interval[1] for interval in intervals) diff --git a/esmvalcore/esgf/_search.py b/esmvalcore/esgf/_search.py index 87d4cdf095..62882a5345 100644 --- a/esmvalcore/esgf/_search.py +++ b/esmvalcore/esgf/_search.py @@ -168,7 +168,7 @@ def select_by_time(files, timerange): for file in files: start_date, end_date = _parse_period(timerange) try: - start, end = _get_start_end_date(file.name) + start, end = _get_start_end_date(file) except ValueError: # If start and end year cannot be read from the filename # just select everything. diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 67007e9608..9ebee00073 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -7,7 +7,7 @@ import re from glob import glob from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union import iris import isodate @@ -17,17 +17,19 @@ from .exceptions import RecipeError from .typing import Facets, FacetValue +if TYPE_CHECKING: + from .esgf import ESGFFile + logger = logging.getLogger(__name__) def _get_from_pattern(pattern, date_range_pattern, stem, group): """Get time, date or datetime from date range patterns in file names.""" - # # Next string allows to test that there is an allowed delimiter (or # string start or end) close to date range (or to single date) start_point = end_point = None context = r"(?:^|[-_]|$)" - # + # First check for a block of two potential dates date_range_pattern_with_context = context + date_range_pattern + context daterange = re.search(date_range_pattern_with_context, stem) @@ -37,6 +39,7 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group): date_range_pattern_with_context = (context + date_range_pattern + context) daterange = re.search(date_range_pattern_with_context, stem) + if daterange: start_point = daterange.group(group) end_group = '_'.join([group, 'end']) @@ -59,41 +62,72 @@ def _get_from_pattern(pattern, date_range_pattern, stem, group): return start_point, end_point -def _get_start_end_date(filename): +def _get_start_end_date( + file: str | Path | LocalFile | ESGFFile +) -> tuple[str, str]: """Get the start and end dates as a string from a file name. - Examples of allowed dates : 1980, 198001, 19801231, - 1980123123, 19801231T23, 19801231T2359, 19801231T235959, - 19801231T235959Z (ISO 8601). + Examples of allowed dates: 1980, 198001, 1980-01, 19801231, 1980-12-31, + 1980123123, 19801231T23, 19801231T2359, 19801231T235959, 19801231T235959Z + (ISO 8601). + + Dates must be surrounded by '-', '_' or '.' (the latter is used by CMIP3 + data), or string start or string end (after removing filename suffix). + + Look first for two dates separated by '-', '_' or '_cat_' (the latter is + used by CMIP3 data), then for one single date, and if there are multiple, + for one date at start or end. + + Parameters + ---------- + file: + The file to read the start and end data from. + + Returns + ------- + tuple[str, str] + The start and end date. - Dates must be surrounded by - or _ or string start or string end - (after removing filename suffix). + Raises + ------ + ValueError + Start or end date cannot be determined. - Look first for two dates separated by - or _, then for one single - date, and if they are multiple, for one date at start or end. """ - stem = Path(filename).stem + if hasattr(file, 'name'): # Path, LocalFile, ESGFFile + stem = Path(file.name).stem + else: # str + stem = Path(file).stem + start_date = end_date = None - # + + # Build regex time_pattern = (r"(?P[0-2][0-9]" r"(?P[0-5][0-9]" r"(?P[0-5][0-9])?)?Z?)") date_pattern = (r"(?P[0-9]{4})" - r"(?P[01][0-9]" - r"(?P[0-3][0-9]" + r"(?P-?[01][0-9]" + r"(?P-?[0-3][0-9]" rf"(T?{time_pattern})?)?)?") datetime_pattern = (rf"(?P{date_pattern})") - # end_datetime_pattern = datetime_pattern.replace(">", "_end>") - date_range_pattern = datetime_pattern + r"[-_]" + end_datetime_pattern + + # Dates can either be delimited by '-', '_', or '_cat_' (the latter for + # CMIP3) + date_range_pattern = ( + datetime_pattern + r"[-_](?:cat_)?" + end_datetime_pattern + ) + + # Find dates using the regex start_date, end_date = _get_from_pattern(datetime_pattern, date_range_pattern, stem, 'datetime') # As final resort, try to get the dates from the file contents - if (start_date is None or end_date is None) and Path(filename).exists(): - logger.debug("Must load file %s for daterange ", filename) - cubes = iris.load(filename) + if ((start_date is None or end_date is None) and + isinstance(file, (str, Path)) and Path(file).exists()): + logger.debug("Must load file %s for daterange ", file) + cubes = iris.load(file) for cube in cubes: logger.debug(cube) @@ -109,12 +143,30 @@ def _get_start_end_date(filename): break if start_date is None or end_date is None: - raise ValueError(f'File {filename} dates do not match a recognized ' - 'pattern and time can not be read from the file') + raise ValueError( + f"File {file} datetimes do not match a recognized pattern and " + f"time coordinate can not be read from the file" + ) + + # Remove potential '-' characters from datetimes + start_date = start_date.replace('-', '') + end_date = end_date.replace('-', '') return start_date, end_date +def _get_start_end_year( + file: str | Path | LocalFile | ESGFFile +) -> tuple[int, int]: + """Get the start and end year as int from a file name. + + See :func:`_get_start_end_date`. + + """ + (start_date, end_date) = _get_start_end_date(file) + return (int(start_date[:4]), int(end_date[:4])) + + def _dates_to_timerange(start_date, end_date): """Convert ``start_date`` and ``end_date`` to ``timerange``. @@ -162,72 +214,6 @@ def _replace_years_with_timerange(variable): variable.pop('end_year', None) -def _get_start_end_year(file): - """Get the start and end year from a file name. - - Examples of allowed dates : 1980, 198001, 19801231, - 1980123123, 19801231T23, 19801231T2359, 19801231T235959, - 19801231T235959Z (ISO 8601). - - Dates must be surrounded by - or _ or string start or string end - (after removing filename suffix). - - Look first for two dates separated by - or _, then for one single - date, and if they are multiple, for one date at start or end. - - Parameters - ---------- - file: LocalFile or esmvalcore.esgf.ESGFFile - The file to read the start and end year from. - - Returns - ------- - tuple[int, int] - The start and end year. - - Raises - ------ - ValueError - When start or end year cannot be determined. - - """ - start_year = end_year = None - - time_pattern = (r"(?P[0-2][0-9]" - r"(?P[0-5][0-9]" - r"(?P[0-5][0-9])?)?Z?)") - date_pattern = (r"(?P[0-9]{4})" - r"(?P[01][0-9]" - r"(?P[0-3][0-9]" - rf"(T?{time_pattern})?)?)?") - - end_date_pattern = date_pattern.replace(">", "_end>") - date_range_pattern = date_pattern + r"[-_]" + end_date_pattern - start_year, end_year = _get_from_pattern(date_pattern, date_range_pattern, - Path(file.name).stem, 'year') - # As final resort, try to get the dates from the file contents - if ((start_year is None or end_year is None) and isinstance(file, Path) - and file.exists()): - logger.debug("Must load file %s for daterange ", file) - cubes = iris.load(file) - - for cube in cubes: - logger.debug(cube) - try: - time = cube.coord('time') - except iris.exceptions.CoordinateNotFoundError: - continue - start_year = time.cell(0).point.year - end_year = time.cell(-1).point.year - break - - if start_year is None or end_year is None: - raise ValueError(f'File {file} dates do not match a recognized ' - 'pattern and time can not be read from the file') - - return int(start_year), int(end_year) - - def _parse_period(timerange): """Parse `timerange` values given as duration periods. diff --git a/tests/unit/local/test_time.py b/tests/unit/local/test_time.py index 57dd8252c7..a01f1b4d05 100644 --- a/tests/unit/local/test_time.py +++ b/tests/unit/local/test_time.py @@ -1,7 +1,11 @@ """Unit tests for time related functions in `esmvalcore.local`.""" +from pathlib import Path + import iris +import pyesgf import pytest +from esmvalcore.esgf import ESGFFile from esmvalcore.local import ( LocalFile, _dates_to_timerange, @@ -11,6 +15,22 @@ _truncate_dates, ) + +def _get_esgf_file(path): + """Get ESGFFile object.""" + result = pyesgf.search.results.FileResult( + json={ + 'dataset_id': 'CMIP6.ABC.v1|something.org', + 'dataset_id_template_': ["%(mip_era)s.%(source_id)s"], + 'project': ['CMIP6'], + 'size': 10, + 'title': path, + }, + context=None, + ) + return ESGFFile([result]) + + FILENAME_CASES = [ ['var_whatever_1980-1981', 1980, 1981], ['var_whatever_1980.nc', 1980, 1980], @@ -31,7 +51,13 @@ 2015, 2015 ], ['pr_A1.186101-200012.nc', 1861, 2000], - ['tas_A1.20C3M_1.CCSM.atmm.1990-01_cat_1999-12.nc', None, None], + ['tas_A1.20C3M_1.CCSM.atmm.1990-01_cat_1999-12.nc', 1990, 1999], + ['E5sf00_1M_1940_032.grb', 1940, 1940], + ['E5sf00_1D_1998-04_167.grb', 1998, 1998], + ['E5sf00_1H_1986-04-11_167.grb', 1986, 1986], + ['E5sf00_1M_1940-1941_032.grb', 1940, 1941], + ['E5sf00_1D_1998-01_1999-12_167.grb', 1998, 1999], + ['E5sf00_1H_2000-01-01_2001-12-31_167.grb', 2000, 2001], ] FILENAME_DATE_CASES = [ @@ -57,7 +83,13 @@ '20150101T000000Z', '20150101T000000Z' ], ['pr_A1.186101-200012.nc', '186101', '200012'], - ['tas_A1.20C3M_1.CCSM.atmm.1990-01_cat_1999-12.nc', None, None], + ['tas_A1.20C3M_1.CCSM.atmm.1990-01_cat_1999-12.nc', '199001', '199912'], + ['E5sf00_1M_1940_032.grb', '1940', '1940'], + ['E5sf00_1D_1998-04_167.grb', '199804', '199804'], + ['E5sf00_1H_1986-04-11_167.grb', '19860411', '19860411'], + ['E5sf00_1M_1940-1941_032.grb', '1940', '1941'], + ['E5sf00_1D_1998-01_1999-12_167.grb', '199801', '199912'], + ['E5sf00_1H_2000-01-01_2001-12-31_167.grb', '20000101', '20011231'], ] @@ -65,38 +97,68 @@ def test_get_start_end_year(case): """Tests for _get_start_end_year function.""" filename, case_start, case_end = case - filename = LocalFile(filename) + + # If the filename is inconclusive or too difficult we resort to reading the + # file, which fails here because the file is not there. if case_start is None and case_end is None: - # If the filename is inconclusive or too difficult - # we resort to reading the file, which fails here - # because the file is not there. with pytest.raises(ValueError): _get_start_end_year(filename) + with pytest.raises(ValueError): + _get_start_end_year(Path(filename)) + with pytest.raises(ValueError): + _get_start_end_year(LocalFile(filename)) + with pytest.raises(ValueError): + _get_start_end_year(_get_esgf_file(filename)) + else: start, end = _get_start_end_year(filename) assert case_start == start assert case_end == end + start, end = _get_start_end_year(Path(filename)) + assert case_start == start + assert case_end == end + start, end = _get_start_end_year(LocalFile(filename)) + assert case_start == start + assert case_end == end + start, end = _get_start_end_year(_get_esgf_file(filename)) + assert case_start == start + assert case_end == end @pytest.mark.parametrize('case', FILENAME_DATE_CASES) def test_get_start_end_date(case): """Tests for _get_start_end_date function.""" filename, case_start, case_end = case - filename = LocalFile(filename) + + # If the filename is inconclusive or too difficult we resort to reading the + # file, which fails here because the file is not there. if case_start is None and case_end is None: - # If the filename is inconclusive or too difficult - # we resort to reading the file, which fails here - # because the file is not there. with pytest.raises(ValueError): _get_start_end_date(filename) + with pytest.raises(ValueError): + _get_start_end_date(Path(filename)) + with pytest.raises(ValueError): + _get_start_end_date(LocalFile(filename)) + with pytest.raises(ValueError): + _get_start_end_date(_get_esgf_file(filename)) + else: start, end = _get_start_end_date(filename) assert case_start == start assert case_end == end + start, end = _get_start_end_date(Path(filename)) + assert case_start == start + assert case_end == end + start, end = _get_start_end_date(LocalFile(filename)) + assert case_start == start + assert case_end == end + start, end = _get_start_end_date(_get_esgf_file(filename)) + assert case_start == start + assert case_end == end -def test_read_time_from_cube(monkeypatch, tmp_path): - """Try to get time from cube if no date in filename.""" +def test_read_years_from_cube(monkeypatch, tmp_path): + """Try to get years from cube if no date in filename.""" monkeypatch.chdir(tmp_path) temp_file = LocalFile('test.nc') cube = iris.cube.Cube([0, 0], var_name='var') @@ -111,7 +173,7 @@ def test_read_time_from_cube(monkeypatch, tmp_path): def test_read_datetime_from_cube(monkeypatch, tmp_path): - """Try to get time from cube if no date in filename.""" + """Try to get datetime from cube if no date in filename.""" monkeypatch.chdir(tmp_path) temp_file = 'test.nc' cube = iris.cube.Cube([0, 0], var_name='var') @@ -133,12 +195,15 @@ def test_raises_if_unable_to_deduce(monkeypatch, tmp_path): iris.save(cube, temp_file) with pytest.raises(ValueError): _get_start_end_date(temp_file) + with pytest.raises(ValueError): + _get_start_end_year(temp_file) def test_fails_if_no_date_present(): """Test raises if no date is present.""" - with pytest.raises((ValueError, OSError)): + with pytest.raises((ValueError)): _get_start_end_date('var_whatever') + with pytest.raises((ValueError)): _get_start_end_year('var_whatever') From 469fd0915e3a04e0c93423b4a23789c87fb9ea1d Mon Sep 17 00:00:00 2001 From: sloosvel <45196700+sloosvel@users.noreply.github.com> Date: Fri, 6 Oct 2023 09:05:49 +0200 Subject: [PATCH 20/72] Relax concatenation checks for `--check_level=relax` and `--check_level=ignore` (#2144) Co-authored-by: Valeriu Predoi --- doc/quickstart/run.rst | 12 +- esmvalcore/dataset.py | 4 +- esmvalcore/preprocessor/_io.py | 277 ++++++++++-------- .../preprocessor/_io/test_concatenate.py | 155 +++++----- tests/unit/test_dataset.py | 4 +- 5 files changed, 251 insertions(+), 201 deletions(-) diff --git a/doc/quickstart/run.rst b/doc/quickstart/run.rst index 03eec25444..ebde6d4075 100644 --- a/doc/quickstart/run.rst +++ b/doc/quickstart/run.rst @@ -78,7 +78,9 @@ or This feature is available for projects that are hosted on the ESGF, i.e. CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs. -To control the strictness of the CMOR checker, use the flag ``--check_level``: +To control the strictness of the CMOR checker and the checks during concatenation +on auxiliary coordinates, supplementary variables, and derived coordinates, +use the flag ``--check_level``: .. code:: bash @@ -86,10 +88,10 @@ To control the strictness of the CMOR checker, use the flag ``--check_level``: Possible values are: - - `ignore`: all errors will be reported as warnings - - `relaxed`: only fail if there are critical errors - - `default`: fail if there are any errors - - `strict`: fail if there are any warnings + - `ignore`: all errors will be reported as warnings. Concatenation will be performed without checks. + - `relaxed`: only fail if there are critical errors. Concatenation will be performed without checks. + - `default`: fail if there are any errors. + - `strict`: fail if there are any warnings. To re-use pre-processed files from a previous run of the same recipe, you can use diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 2b2230a59c..d4bd665aa6 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -736,7 +736,9 @@ def _load(self) -> Cube: 'session': self.session, **self.facets, } - settings['concatenate'] = {} + settings['concatenate'] = { + 'check_level': self.session['check_level'] + } settings['cmor_check_metadata'] = { 'check_level': self.session['check_level'], 'cmor_table': self.facets['project'], diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 4da6736501..78833cc4ab 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -12,14 +12,17 @@ import iris import iris.aux_factory import iris.exceptions +import isodate +import numpy as np import yaml from cf_units import suppress_errors from iris.cube import CubeList +from esmvalcore.cmor.check import CheckLevels from esmvalcore.iris_helpers import merge_cube_attributes from .._task import write_ncl_settings -from ._time import extract_time +from ._time import clip_timerange logger = logging.getLogger(__name__) @@ -95,10 +98,8 @@ def _get_attr_from_field_coord(ncfield, coord_name, attr): def _load_callback(raw_cube, field, _): """Use this callback to fix anything Iris tries to break.""" # Remove attributes that cause issues with merging and concatenation - _delete_attributes( - raw_cube, - ('creation_date', 'tracking_id', 'history', 'comment') - ) + _delete_attributes(raw_cube, + ('creation_date', 'tracking_id', 'history', 'comment')) for coord in raw_cube.coords(): # Iris chooses to change longitude and latitude units to degrees # regardless of value in file, so reinstating file value @@ -184,17 +185,110 @@ def load( return raw_cubes -def _by_two_concatenation(cubes): - """Perform a by-2 concatenation to avoid gaps.""" - concatenated = iris.cube.CubeList(cubes).concatenate() - if len(concatenated) == 1: - return concatenated[0] +def _concatenate_cubes(cubes, check_level): + """Concatenate cubes according to the check_level.""" + kwargs = { + 'check_aux_coords': True, + 'check_cell_measures': True, + 'check_ancils': True, + 'check_derived_coords': True + } - concatenated = _concatenate_overlapping_cubes(concatenated) - if len(concatenated) == 2: - _get_concatenation_error(concatenated) - else: - return concatenated[0] + if check_level > CheckLevels.DEFAULT: + kwargs = dict.fromkeys(kwargs, False) + logger.debug( + 'Concatenation will be performed without checking ' + 'auxiliary coordinates, cell measures, ancillaries ' + 'and derived coordinates present in the cubes.', ) + + concatenated = iris.cube.CubeList(cubes).concatenate(**kwargs) + + return concatenated + + +def _check_time_overlaps(cubes): + """Handle time overlaps.""" + times = [cube.coord('time').core_points() for cube in cubes] + for index, _ in enumerate(times[:-1]): + overlap = np.intersect1d(times[index], times[index + 1]) + if overlap.size != 0: + overlapping_cubes = cubes[index:index + 2] + time_1 = overlapping_cubes[0].coord('time').core_points() + time_2 = overlapping_cubes[1].coord('time').core_points() + + # case 1: both cubes start at the same time -> return longer cube + if time_1[0] == time_2[0]: + if time_1[-1] <= time_2[-1]: + cubes.pop(index) + discarded_cube_index = 0 + used_cube_index = 1 + else: + cubes.pop(index + 1) + discarded_cube_index = 1 + used_cube_index = 0 + logger.debug( + "Both cubes start at the same time but cube %s " + "ends before %s", + overlapping_cubes[discarded_cube_index], + overlapping_cubes[used_cube_index], + ) + logger.debug( + "Cube %s contains all needed data so using it fully", + overlapping_cubes[used_cube_index], + ) + + # case 2: cube1 starts before cube2 + # case 2.1: cube1 ends after cube2 -> return cube1 + elif time_1[-1] > time_2[-1]: + cubes.pop(index + 1) + logger.debug("Using only data from %s", overlapping_cubes[0]) + + # case 2.2: cube1 ends before cube2 -> use full cube2 + # and shorten cube1 + else: + new_time = np.delete( + time_1, + np.argwhere(np.in1d(time_1, overlap)), + ) + new_dates = overlapping_cubes[0].coord('time').units.num2date( + new_time) + logger.debug( + "Extracting time slice between %s and %s from cube %s " + "to use it for concatenation with cube %s", + new_dates[0], + new_dates[-1], + overlapping_cubes[0], + overlapping_cubes[1], + ) + + start_point = isodate.date_isoformat( + new_dates[0], format=isodate.isostrf.DATE_BAS_COMPLETE) + end_point = isodate.date_isoformat( + new_dates[-1], format=isodate.isostrf.DATE_BAS_COMPLETE) + new_cube = clip_timerange(overlapping_cubes[0], + f'{start_point}/{end_point}') + + cubes[index] = new_cube + return cubes + + +def _fix_calendars(cubes): + """Check and homogenise calendars, if possible.""" + calendars = [cube.coord('time').units.calendar for cube in cubes] + unique_calendars = np.unique(calendars) + + calendar_ocurrences = np.array( + [calendars.count(calendar) for calendar in unique_calendars]) + calendar_index = int( + np.argwhere(calendar_ocurrences == calendar_ocurrences.max())) + + for cube in cubes: + time_coord = cube.coord('time') + old_calendar = time_coord.units.calendar + if old_calendar != unique_calendars[calendar_index]: + new_unit = time_coord.units.change_calendar( + unique_calendars[calendar_index]) + time_coord.units = new_unit def _get_concatenation_error(cubes): @@ -214,28 +308,56 @@ def _get_concatenation_error(cubes): raise ValueError(f'Can not concatenate cubes: {msg}') -def concatenate(cubes): - """Concatenate all cubes after fixing metadata.""" +def _sort_cubes_by_time(cubes): + """Sort CubeList by time coordinate.""" + try: + cubes = sorted(cubes, key=lambda c: c.coord("time").cell(0).point) + except iris.exceptions.CoordinateNotFoundError as exc: + msg = "One or more cubes {} are missing".format(cubes) + \ + " time coordinate: {}".format(str(exc)) + raise ValueError(msg) + except TypeError as error: + msg = ("Cubes cannot be sorted " + f"due to differing time units: {str(error)}") + raise TypeError(msg) from error + return cubes + + +def concatenate(cubes, check_level=CheckLevels.DEFAULT): + """Concatenate all cubes after fixing metadata. + + Parameters + ---------- + cubes: iterable of iris.cube.Cube + Data cubes to be concatenated + check_level: CheckLevels + Level of strictness of the checks in the concatenation. + + Returns + ------- + cube: iris.cube.Cube + Resulting concatenated cube. + + Raises + ------ + ValueError + Concatenation was not possible. + """ if not cubes: return cubes if len(cubes) == 1: return cubes[0] merge_cube_attributes(cubes) + cubes = _sort_cubes_by_time(cubes) + _fix_calendars(cubes) + cubes = _check_time_overlaps(cubes) + result = _concatenate_cubes(cubes, check_level=check_level) - if len(cubes) > 1: - # order cubes by first time point - try: - cubes = sorted(cubes, key=lambda c: c.coord("time").cell(0).point) - except iris.exceptions.CoordinateNotFoundError as exc: - msg = "One or more cubes {} are missing".format(cubes) + \ - " time coordinate: {}".format(str(exc)) - raise ValueError(msg) - - # iteratively concatenate starting with first cube - result = cubes[0] - for cube in cubes[1:]: - result = _by_two_concatenation([result, cube]) + if len(result) == 1: + result = result[0] + else: + _get_concatenation_error(result) _fix_aux_factories(result) @@ -410,98 +532,3 @@ def _write_ncl_metadata(output_dir, metadata): write_ncl_settings(info, filename) return filename - - -def _concatenate_overlapping_cubes(cubes): - """Concatenate time-overlapping cubes (two cubes only).""" - # we arrange [cube1, cube2] so that cube1.start <= cube2.start - if cubes[0].coord('time').points[0] <= cubes[1].coord('time').points[0]: - cubes = [cubes[0], cubes[1]] - logger.debug( - "Will attempt to concatenate cubes %s " - "and %s in this order", cubes[0], cubes[1]) - else: - cubes = [cubes[1], cubes[0]] - logger.debug( - "Will attempt to concatenate cubes %s " - "and %s in this order", cubes[1], cubes[0]) - - # get time end points - time_1 = cubes[0].coord('time') - time_2 = cubes[1].coord('time') - if time_1.units != time_2.units: - raise ValueError( - f"Cubes\n{cubes[0]}\nand\n{cubes[1]}\ncan not be concatenated: " - f"time units {time_1.units}, calendar {time_1.units.calendar} " - f"and {time_2.units}, calendar {time_2.units.calendar} differ") - data_start_1 = time_1.cell(0).point - data_start_2 = time_2.cell(0).point - data_end_1 = time_1.cell(-1).point - data_end_2 = time_2.cell(-1).point - - # case 1: both cubes start at the same time -> return longer cube - if data_start_1 == data_start_2: - if data_end_1 <= data_end_2: - logger.debug( - "Both cubes start at the same time but cube %s " - "ends before %s", cubes[0], cubes[1]) - logger.debug("Cube %s contains all needed data so using it fully", - cubes[1]) - cubes = [cubes[1]] - else: - logger.debug( - "Both cubes start at the same time but cube %s " - "ends before %s", cubes[1], cubes[0]) - logger.debug("Cube %s contains all needed data so using it fully", - cubes[0]) - cubes = [cubes[0]] - - # case 2: cube1 starts before cube2 - else: - # find time overlap, if any - start_overlap = next((time_1.units.num2date(t) - for t in time_1.points if t in time_2.points), - None) - # case 2.0: no overlap (new iris implementation does allow - # concatenation of cubes with no overlap) - if not start_overlap: - logger.debug( - "Unable to concatenate non-overlapping cubes\n%s\nand\n%s" - "separated in time.", cubes[0], cubes[1]) - # case 2.1: cube1 ends after cube2 -> return cube1 - elif data_end_1 > data_end_2: - cubes = [cubes[0]] - logger.debug("Using only data from %s", cubes[0]) - # case 2.2: cube1 ends before cube2 -> use full cube2 and shorten cube1 - else: - logger.debug( - "Extracting time slice between %s and %s from cube %s to use " - "it for concatenation with cube %s", "-".join([ - str(data_start_1.year), - str(data_start_1.month), - str(data_start_1.day) - ]), "-".join([ - str(start_overlap.year), - str(start_overlap.month), - str(start_overlap.day) - ]), cubes[0], cubes[1]) - c1_delta = extract_time(cubes[0], data_start_1.year, - data_start_1.month, data_start_1.day, - start_overlap.year, start_overlap.month, - start_overlap.day) - # convert c1_delta scalar cube to vector cube, if needed - if c1_delta.shape == (): - c1_delta = iris.util.new_axis(c1_delta, scalar_coord="time") - cubes = iris.cube.CubeList([c1_delta, cubes[1]]) - logger.debug("Attempting concatenatenation of %s with %s", - c1_delta, cubes[1]) - try: - cubes = [iris.cube.CubeList(cubes).concatenate_cube()] - except iris.exceptions.ConcatenateError as ex: - logger.error('Can not concatenate cubes: %s', ex) - logger.error('Cubes:') - for cube in cubes: - logger.error(cube) - raise ex - - return cubes diff --git a/tests/integration/preprocessor/_io/test_concatenate.py b/tests/integration/preprocessor/_io/test_concatenate.py index a5fef83380..3171c27a25 100644 --- a/tests/integration/preprocessor/_io/test_concatenate.py +++ b/tests/integration/preprocessor/_io/test_concatenate.py @@ -15,21 +15,25 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList +from esmvalcore.cmor.check import CheckLevels from esmvalcore.preprocessor import _io def get_hybrid_pressure_cube(): """Return cube with hybrid pressure coordinate.""" ap_coord = AuxCoord([1.0], bounds=[[0.0, 2.0]], var_name='ap', units='Pa') - b_coord = AuxCoord([0.0], bounds=[[-0.5, 1.5]], - var_name='b', units=Unit('1')) + b_coord = AuxCoord([0.0], + bounds=[[-0.5, 1.5]], + var_name='b', + units=Unit('1')) ps_coord = AuxCoord([[[100000]]], var_name='ps', units='Pa') x_coord = AuxCoord( 0.0, var_name='x', standard_name='atmosphere_hybrid_sigma_pressure_coordinate', ) - cube = Cube([[[[0.0]]]], var_name='x', + cube = Cube([[[[0.0]]]], + var_name='x', aux_coords_and_dims=[(ap_coord, 1), (b_coord, 1), (ps_coord, (0, 2, 3)), (x_coord, ())]) return cube @@ -54,7 +58,9 @@ def get_hybrid_pressure_cube_list(): def get_time_coord(time_point): """Time coordinate.""" - return DimCoord([time_point], var_name='time', standard_name='time', + return DimCoord([time_point], + var_name='time', + standard_name='time', units='days since 6453-2-1') @@ -73,11 +79,14 @@ def mock_atmosphere_sigma_cube(): """Return mocked cube with atmosphere sigma coordinate.""" cube = unittest.mock.create_autospec(Cube, spec_set=True, instance=True) ptop_coord = AuxCoord([1.0], var_name='ptop', units='Pa') - lev_coord = AuxCoord([0.0], bounds=[[-0.5, 1.5]], var_name='lev', + lev_coord = AuxCoord([0.0], + bounds=[[-0.5, 1.5]], + var_name='lev', units='1') ps_coord = AuxCoord([[[100000]]], var_name='ps', units='Pa') - cube.coord.side_effect = [ptop_coord, lev_coord, ps_coord, - ptop_coord, lev_coord, ps_coord] + cube.coord.side_effect = [ + ptop_coord, lev_coord, ps_coord, ptop_coord, lev_coord, ps_coord + ] cube.coords.return_value = [ ptop_coord, lev_coord, @@ -100,8 +109,9 @@ def mock_hybrid_height_cube(): lev_coord = AuxCoord([1.0], bounds=[[0.0, 2.0]], var_name='lev', units='m') b_coord = AuxCoord([0.0], bounds=[[-0.5, 1.5]], var_name='b') orog_coord = AuxCoord([[[100000]]], var_name='orog', units='m') - cube.coord.side_effect = [lev_coord, b_coord, orog_coord, - lev_coord, b_coord, orog_coord] + cube.coord.side_effect = [ + lev_coord, b_coord, orog_coord, lev_coord, b_coord, orog_coord + ] cube.coords.return_value = [ lev_coord, b_coord, @@ -122,11 +132,14 @@ def mock_hybrid_pressure_cube(): """Return mocked cube with hybrid pressure coordinate.""" cube = unittest.mock.create_autospec(Cube, spec_set=True, instance=True) ap_coord = AuxCoord([1.0], bounds=[[0.0, 2.0]], var_name='ap', units='Pa') - b_coord = AuxCoord([0.0], bounds=[[-0.5, 1.5]], - var_name='b', units=Unit('1')) + b_coord = AuxCoord([0.0], + bounds=[[-0.5, 1.5]], + var_name='b', + units=Unit('1')) ps_coord = AuxCoord([[[100000]]], var_name='ps', units='Pa') - cube.coord.side_effect = [ap_coord, b_coord, ps_coord, - ap_coord, b_coord, ps_coord] + cube.coord.side_effect = [ + ap_coord, b_coord, ps_coord, ap_coord, b_coord, ps_coord + ] cube.coords.return_value = [ ap_coord, b_coord, @@ -182,9 +195,10 @@ def test_fix_aux_factories_atmosphere_sigma(mock_atmosphere_sigma_cube): # Test with aux_factory object _io._fix_aux_factories(mock_atmosphere_sigma_cube) mock_atmosphere_sigma_cube.coords.assert_called_once_with() - mock_atmosphere_sigma_cube.coord.assert_has_calls([call(var_name='ptop'), - call(var_name='lev'), - call(var_name='ps')]) + mock_atmosphere_sigma_cube.coord.assert_has_calls( + [call(var_name='ptop'), + call(var_name='lev'), + call(var_name='ps')]) mock_atmosphere_sigma_cube.add_aux_factory.assert_not_called() # Test without aux_factory object @@ -192,9 +206,10 @@ def test_fix_aux_factories_atmosphere_sigma(mock_atmosphere_sigma_cube): mock_atmosphere_sigma_cube.aux_factories = ['dummy'] _io._fix_aux_factories(mock_atmosphere_sigma_cube) mock_atmosphere_sigma_cube.coords.assert_called_once_with() - mock_atmosphere_sigma_cube.coord.assert_has_calls([call(var_name='ptop'), - call(var_name='lev'), - call(var_name='ps')]) + mock_atmosphere_sigma_cube.coord.assert_has_calls( + [call(var_name='ptop'), + call(var_name='lev'), + call(var_name='ps')]) mock_atmosphere_sigma_cube.add_aux_factory.assert_called_once() @@ -205,9 +220,10 @@ def test_fix_aux_factories_hybrid_height(mock_hybrid_height_cube): # Test with aux_factory object _io._fix_aux_factories(mock_hybrid_height_cube) mock_hybrid_height_cube.coords.assert_called_once_with() - mock_hybrid_height_cube.coord.assert_has_calls([call(var_name='lev'), - call(var_name='b'), - call(var_name='orog')]) + mock_hybrid_height_cube.coord.assert_has_calls( + [call(var_name='lev'), + call(var_name='b'), + call(var_name='orog')]) mock_hybrid_height_cube.add_aux_factory.assert_not_called() # Test without aux_factory object @@ -215,9 +231,10 @@ def test_fix_aux_factories_hybrid_height(mock_hybrid_height_cube): mock_hybrid_height_cube.aux_factories = ['dummy'] _io._fix_aux_factories(mock_hybrid_height_cube) mock_hybrid_height_cube.coords.assert_called_once_with() - mock_hybrid_height_cube.coord.assert_has_calls([call(var_name='lev'), - call(var_name='b'), - call(var_name='orog')]) + mock_hybrid_height_cube.coord.assert_has_calls( + [call(var_name='lev'), + call(var_name='b'), + call(var_name='orog')]) mock_hybrid_height_cube.add_aux_factory.assert_called_once() @@ -228,9 +245,10 @@ def test_fix_aux_factories_hybrid_pressure(mock_hybrid_pressure_cube): # Test with aux_factory object _io._fix_aux_factories(mock_hybrid_pressure_cube) mock_hybrid_pressure_cube.coords.assert_called_once_with() - mock_hybrid_pressure_cube.coord.assert_has_calls([call(var_name='ap'), - call(var_name='b'), - call(var_name='ps')]) + mock_hybrid_pressure_cube.coord.assert_has_calls( + [call(var_name='ap'), + call(var_name='b'), + call(var_name='ps')]) mock_hybrid_pressure_cube.add_aux_factory.assert_not_called() # Test without aux_factory object @@ -238,9 +256,10 @@ def test_fix_aux_factories_hybrid_pressure(mock_hybrid_pressure_cube): mock_hybrid_pressure_cube.aux_factories = ['dummy'] _io._fix_aux_factories(mock_hybrid_pressure_cube) mock_hybrid_pressure_cube.coords.assert_called_once_with() - mock_hybrid_pressure_cube.coord.assert_has_calls([call(var_name='ap'), - call(var_name='b'), - call(var_name='ps')]) + mock_hybrid_pressure_cube.coord.assert_has_calls( + [call(var_name='ap'), + call(var_name='b'), + call(var_name='ps')]) mock_hybrid_pressure_cube.add_aux_factory.assert_called_once() @@ -250,8 +269,10 @@ def test_fix_aux_factories_real_cube(real_hybrid_pressure_cube): assert not real_hybrid_pressure_cube.coords('air_pressure') _io._fix_aux_factories(real_hybrid_pressure_cube) air_pressure_coord = real_hybrid_pressure_cube.coord('air_pressure') - expected_coord = AuxCoord([[[[1.0]]]], bounds=[[[[[-50000., 150002.]]]]], - standard_name='air_pressure', units='Pa') + expected_coord = AuxCoord([[[[1.0]]]], + bounds=[[[[[-50000., 150002.]]]]], + standard_name='air_pressure', + units='Pa') assert air_pressure_coord == expected_coord @@ -268,6 +289,18 @@ def test_concatenation_with_aux_factory(real_hybrid_pressure_cube_list): assert air_pressure_coord == expected_coord +@pytest.mark.parametrize('check_level', + [CheckLevels.RELAXED, CheckLevels.IGNORE]) +def test_relax_concatenation(check_level, caplog): + caplog.set_level('DEBUG') + cubes = get_hybrid_pressure_cube_list() + _io.concatenate(cubes, check_level) + msg = ('Concatenation will be performed without checking ' + 'auxiliary coordinates, cell measures, ancillaries ' + 'and derived coordinates present in the cubes.') + assert msg in caplog.text + + class TestConcatenate(unittest.TestCase): """Tests for :func:`esmvalcore.preprocessor._io.concatenate`.""" @@ -306,7 +339,7 @@ def test_concatenate_noop(self): np.testing.assert_array_equal( concatenated.coord('time').points, np.array([1, 2])) - def test_concatenate_with_overlap(self): + def test_concatenate_with_overlap(self, ): """Test concatenation of time overalapping cubes.""" self._add_cube([6.5, 7.5], [6., 7.]) concatenated = _io.concatenate(self.raw_cubes) @@ -439,9 +472,26 @@ def test_concatenate_differing_attributes(self): concatenated.coord('time').points, np.array([1, 2, 3, 4, 5, 6])) self.assertEqual( concatenated.attributes, - {'equal_attr': 1, 'different_attr': '1 2 3'}, + { + 'equal_attr': 1, + 'different_attr': '1 2 3' + }, ) + def test_convert_calendar_concatenate_with_overlap(self): + """Test compatible calendars get converted.""" + time_coord = DimCoord([4., 5.], + var_name='time', + standard_name='time', + units=Unit('days since 1950-01-01', + calendar='proleptic_gregorian')) + self.raw_cubes.append( + Cube([33., 55.], + var_name='sample', + dim_coords_and_dims=((time_coord, 0), ))) + concatenated = _io.concatenate(self.raw_cubes) + assert concatenated.coord('time').units.calendar == 'standard' + def test_fail_on_calendar_concatenate_with_overlap(self): """Test fail of concatenation with overlap.""" time_coord = DimCoord([3., 7000.], @@ -453,42 +503,9 @@ def test_fail_on_calendar_concatenate_with_overlap(self): Cube([33., 55.], var_name='sample', dim_coords_and_dims=((time_coord, 0), ))) - with self.assertRaises((TypeError, ValueError)): + with self.assertRaises(TypeError): _io.concatenate(self.raw_cubes) - def test_fail_on_units_concatenate_with_overlap(self): - """Test fail of concatenation with overlap.""" - time_coord_1 = DimCoord([3., 7000.], - var_name='time', - standard_name='time', - units=Unit('days since 1950-01-01', - calendar='360_day')) - time_coord_2 = DimCoord([3., 9000.], - var_name='time', - standard_name='time', - units=Unit('days since 1950-01-01', - calendar='360_day')) - time_coord_3 = DimCoord([3., 9000.], - var_name='time', - standard_name='time', - units=Unit('days since 1850-01-01', - calendar='360_day')) - raw_cubes = [] - raw_cubes.append( - Cube([33., 55.], - var_name='sample', - dim_coords_and_dims=((time_coord_1, 0), ))) - raw_cubes.append( - Cube([33., 55.], - var_name='sample', - dim_coords_and_dims=((time_coord_2, 0), ))) - raw_cubes.append( - Cube([33., 55.], - var_name='sample', - dim_coords_and_dims=((time_coord_3, 0), ))) - with self.assertRaises(ValueError): - _io.concatenate(raw_cubes) - def test_fail_metadata_differs(self): """Test exception raised if two cubes have different metadata.""" self.raw_cubes[0].units = 'm' diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index f96161d4cf..951f3f17f8 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1755,7 +1755,9 @@ def mock_preprocess(items, step, input_files, output_file, debug, 'short_name': 'chl', 'frequency': 'yr', }, - 'concatenate': {}, + 'concatenate': { + 'check_level': CheckLevels.DEFAULT, + }, 'add_supplementary_variables': { 'supplementary_cubes': [], }, From adeb1e27ce999d48a67b6befef72c7baf3ceb20c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 9 Oct 2023 11:37:58 +0100 Subject: [PATCH 21/72] Add file encoding (and some read modes) at open file step (#2219) Co-authored-by: Bouwe Andela --- doc/gensidebar.py | 4 ++-- esmvalcore/_citation.py | 8 +++++--- esmvalcore/_main.py | 4 ++-- esmvalcore/_task.py | 10 +++++----- esmvalcore/cmor/table.py | 12 ++++++------ esmvalcore/config/_config.py | 2 +- esmvalcore/config/_config_object.py | 2 +- esmvalcore/config/_diagnostics.py | 2 +- esmvalcore/config/_esgf_pyclient.py | 2 +- esmvalcore/config/_logging.py | 2 +- esmvalcore/esgf/_download.py | 4 ++-- esmvalcore/experimental/recipe.py | 3 ++- esmvalcore/experimental/recipe_info.py | 2 +- esmvalcore/experimental/recipe_output.py | 6 +++--- esmvalcore/preprocessor/_io.py | 2 +- setup.py | 6 +++--- tests/integration/cmor/_fixes/icon/test_icon.py | 2 +- tests/integration/esgf/test_search_download.py | 2 +- tests/integration/recipe/test_recipe.py | 2 +- tests/integration/test_citation.py | 6 +++--- tests/integration/test_diagnostic_run.py | 4 ++-- tests/integration/test_local.py | 6 ++++-- tests/integration/test_main.py | 2 +- tests/integration/test_task.py | 4 ++-- tests/unit/config/test_config.py | 4 ++-- tests/unit/config/test_esgf_pyclient.py | 4 ++-- tests/unit/documentation/test_changelog.py | 2 +- tests/unit/esgf/test_download.py | 4 ++-- tests/unit/task/test_diagnostic_task.py | 4 ++-- tests/unit/task/test_resume_task.py | 2 +- 30 files changed, 62 insertions(+), 57 deletions(-) diff --git a/doc/gensidebar.py b/doc/gensidebar.py index 970722ff0a..01f8b3e839 100644 --- a/doc/gensidebar.py +++ b/doc/gensidebar.py @@ -10,13 +10,13 @@ def _write_if_changed(fname, contents): """Write/update file only if changed.""" try: - with open(fname, "r") as stream: + with open(fname, "r", encoding="utf-8") as stream: old_contents = stream.read() except IOError: old_contents = "" if old_contents != contents: - with open(fname, "w") as stream: + with open(fname, "w", encoding="utf-8") as stream: stream.write(contents) diff --git a/esmvalcore/_citation.py b/esmvalcore/_citation.py index 5637f115b3..510aa8cb42 100644 --- a/esmvalcore/_citation.py +++ b/esmvalcore/_citation.py @@ -101,7 +101,8 @@ def _save_citation_bibtex(product_name, tags, json_urls): entries.add(cmip_citation) citation_entries.extend(sorted(entries)) - with open(f'{product_name}_citation.bibtex', 'w') as file: + with open(f'{product_name}_citation.bibtex', + 'w', encoding='utf-8') as file: file.write('\n'.join(citation_entries)) @@ -126,7 +127,8 @@ def _save_citation_info_txt(product_name, info_urls, other_info): for t in sorted(other_info)) if lines: - with open(f'{product_name}_data_citation_info.txt', 'w') as file: + with open(f'{product_name}_data_citation_info.txt', + 'w', encoding='utf-8') as file: file.write('\n'.join(lines) + '\n') @@ -196,7 +198,7 @@ def _collect_bibtex_citation(tag): """Collect information from bibtex files.""" bibtex_file = DIAGNOSTICS.references / f'{tag}.bibtex' if bibtex_file.is_file(): - entry = bibtex_file.read_text() + entry = bibtex_file.read_text(encoding='utf-8') else: entry = '' logger.warning( diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 760ec054bd..c802dafad3 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -64,10 +64,10 @@ def parse_resume(resume, recipe): resume[i] = Path(os.path.expandvars(resume_dir)).expanduser() # Sanity check resume directories: - current_recipe = recipe.read_text() + current_recipe = recipe.read_text(encoding='utf-8') for resume_dir in resume: resume_recipe = resume_dir / 'run' / recipe.name - if current_recipe != resume_recipe.read_text(): + if current_recipe != resume_recipe.read_text(encoding='utf-8'): raise ValueError(f'Only identical recipes can be resumed, but ' f'{resume_recipe} is different from {recipe}') return resume diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 01d7861f28..04200371cd 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -119,7 +119,7 @@ def _log_resource_usage(): """Write resource usage to file.""" process = psutil.Process(pid) start_time = time.time() - with open(filename, 'w') as file: + with open(filename, 'w', encoding='utf-8') as file: for msg, max_mem in _get_resource_usage(process, start_time, children): file.write(msg) @@ -219,7 +219,7 @@ def _ncl_type(value): 'end if\n'.format(var_name=var_name)) lines.append(_py2ncl(value, var_name)) - with open(filename, mode) as file: + with open(filename, mode, encoding='utf-8') as file: file.write('\n'.join(lines)) file.write('\n') @@ -301,7 +301,7 @@ def __init__(self, prev_preproc_dir, preproc_dir, name): # Reconstruct output prev_metadata_file = prev_preproc_dir / 'metadata.yml' - with prev_metadata_file.open('rb') as file: + with prev_metadata_file.open('r', encoding='utf-8') as file: prev_metadata = yaml.safe_load(file) products = set() @@ -323,7 +323,7 @@ def _run(self, _): # Write metadata to file self._metadata_file.parent.mkdir(parents=True) - with self._metadata_file.open('w') as file: + with self._metadata_file.open('w', encoding='utf-8') as file: yaml.safe_dump(metadata, file) return [str(self._metadata_file)] @@ -609,7 +609,7 @@ def _collect_provenance(self): logger.debug("Collecting provenance from %s", provenance_file) start = time.time() - table = yaml.safe_load(provenance_file.read_text()) + table = yaml.safe_load(provenance_file.read_text(encoding='utf-8')) ignore = ( 'auxiliary_data_dir', diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 01659aa206..aea873a9de 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -150,7 +150,7 @@ def _read_cmor_tables(cfg_file: Path, mtime: float) -> dict[str, CMORTable]: cfg_developer = yaml.safe_load(file) cwd = os.path.dirname(os.path.realpath(__file__)) var_alt_names_file = os.path.join(cwd, 'variable_alt_names.yml') - with open(var_alt_names_file, 'r') as yfile: + with open(var_alt_names_file, 'r', encoding='utf-8') as yfile: alt_names = yaml.safe_load(yfile) cmor_tables: dict[str, CMORTable] = {} @@ -415,7 +415,7 @@ def _get_cmor_path(cmor_tables_path): 'CMOR tables not found in {}'.format(cmor_tables_path)) def _load_table(self, json_file): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: raw_data = json.loads(inf.read()) if not self._is_table(raw_data): return @@ -465,7 +465,7 @@ def _load_coordinates(self): self.coords = {} for json_file in glob.glob( os.path.join(self._cmor_folder, '*coordinate*.json')): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: table_data = json.loads(inf.read()) for coord_name in table_data['axis_entry'].keys(): coord = CoordinateInfo(coord_name) @@ -477,7 +477,7 @@ def _load_controlled_vocabulary(self): self.institutes = {} for json_file in glob.glob(os.path.join(self._cmor_folder, '*_CV.json')): - with open(json_file) as inf: + with open(json_file, encoding='utf-8') as inf: table_data = json.loads(inf.read()) try: exps = table_data['CV']['experiment_id'] @@ -846,7 +846,7 @@ def _load_table(self, table_file, table_name=''): self._read_table_file(table_file, table) def _read_table_file(self, table_file, table=None): - with open(table_file) as self._current_table: + with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: key, value = self._last_line_read @@ -1057,7 +1057,7 @@ def get_variable(self, table, short_name, derived=False): return self.tables['custom'].get(short_name, None) def _read_table_file(self, table_file, table=None): - with open(table_file) as self._current_table: + with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: key, value = self._last_line_read diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 85f8f1f1c2..5dcad80e05 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -43,7 +43,7 @@ def _load_extra_facets(project, extra_facets_dir): config_file_paths = config_path.glob(f"{project.lower()}-*.yml") for config_file_path in sorted(config_file_paths): logger.debug("Loading extra facets from %s", config_file_path) - with config_file_path.open() as config_file: + with config_file_path.open(encoding='utf-8') as config_file: config_piece = yaml.safe_load(config_file) if config_piece: _deep_update(config, config_piece) diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index a0fd23e90b..0b9c596177 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -225,7 +225,7 @@ def _read_config_file(config_file): if not config_file.exists(): raise IOError(f'Config file `{config_file}` does not exist.') - with open(config_file, 'r') as file: + with open(config_file, 'r', encoding='utf-8') as file: cfg = yaml.safe_load(file) return cfg diff --git a/esmvalcore/config/_diagnostics.py b/esmvalcore/config/_diagnostics.py index e54dba87a6..c8f0869c9e 100644 --- a/esmvalcore/config/_diagnostics.py +++ b/esmvalcore/config/_diagnostics.py @@ -83,7 +83,7 @@ def from_file(cls, filename: str): """Load the reference tags used for provenance recording.""" if os.path.exists(filename): logger.debug("Loading tags from %s", filename) - with open(filename) as file: + with open(filename, 'r', encoding='utf-8') as file: tags = cls(yaml.safe_load(file)) tags.source_file = filename return tags diff --git a/esmvalcore/config/_esgf_pyclient.py b/esmvalcore/config/_esgf_pyclient.py index 7e80cc4892..1d473068e6 100644 --- a/esmvalcore/config/_esgf_pyclient.py +++ b/esmvalcore/config/_esgf_pyclient.py @@ -110,7 +110,7 @@ def read_config_file(): if mode & stat.S_IRWXG or mode & stat.S_IRWXO: logger.warning("Correcting unsafe permissions on %s", CONFIG_FILE) os.chmod(CONFIG_FILE, stat.S_IRUSR | stat.S_IWUSR) - with CONFIG_FILE.open() as file: + with CONFIG_FILE.open(encoding='utf-8') as file: cfg = yaml.safe_load(file) else: logger.info( diff --git a/esmvalcore/config/_logging.py b/esmvalcore/config/_logging.py index 674fad953d..3763c3b688 100644 --- a/esmvalcore/config/_logging.py +++ b/esmvalcore/config/_logging.py @@ -87,7 +87,7 @@ def configure_logging( cfg_file = Path(cfg_file).absolute() - with open(cfg_file) as file_handler: + with open(cfg_file, 'r', encoding='utf-8') as file_handler: cfg = yaml.safe_load(file_handler) if output_dir is None: diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index 7efe389293..551aacceed 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -52,7 +52,7 @@ def compute_speed(size, duration): def load_speeds(): """Load average download speeds from HOSTS_FILE.""" try: - content = HOSTS_FILE.read_text() + content = HOSTS_FILE.read_text(encoding='utf-8') except FileNotFoundError: content = '{}' speeds = yaml.safe_load(content) @@ -94,7 +94,7 @@ def atomic_write(filename): filename.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile(prefix=f"{filename}.") as file: tmp_file = file.name - with open(tmp_file, 'w') as file: + with open(tmp_file, 'w', encoding='utf-8') as file: yield file shutil.move(tmp_file, filename) diff --git a/esmvalcore/experimental/recipe.py b/esmvalcore/experimental/recipe.py index 5839d72df8..18e520324b 100644 --- a/esmvalcore/experimental/recipe.py +++ b/esmvalcore/experimental/recipe.py @@ -70,7 +70,8 @@ def name(self): def data(self) -> dict: """Return dictionary representation of the recipe.""" if self._data is None: - self._data = yaml.safe_load(open(self.path, 'r')) + with open(self.path, 'r', encoding='utf-8') as yaml_file: + self._data = yaml.safe_load(yaml_file) return self._data def _load(self, session: Session) -> RecipeEngine: diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py index 0e00ea3667..cb0fd32f95 100644 --- a/esmvalcore/experimental/recipe_info.py +++ b/esmvalcore/experimental/recipe_info.py @@ -72,7 +72,7 @@ def _repr_html_(self) -> str: @classmethod def from_yaml(cls, path: str): """Return instance of 'RecipeInfo' from a recipe in yaml format.""" - data = yaml.safe_load(open(path, 'r')) + data = yaml.safe_load(Path(path).read_text(encoding='utf-8')) return cls(data, filename=path) @property diff --git a/esmvalcore/experimental/recipe_output.py b/esmvalcore/experimental/recipe_output.py index bcb38c019a..69f63765a2 100644 --- a/esmvalcore/experimental/recipe_output.py +++ b/esmvalcore/experimental/recipe_output.py @@ -200,7 +200,7 @@ def write_html(self): template = get_template('recipe_output_page.j2') html_dump = self.render(template=template) - with open(filename, 'w') as file: + with open(filename, 'w', encoding='utf-8') as file: file.write(html_dump) logger.info("Wrote recipe output to:\nfile://%s", filename) @@ -225,11 +225,11 @@ def render(self, template=None): def read_main_log(self) -> str: """Read log file.""" - return self.session.main_log.read_text() + return self.session.main_log.read_text(encoding='utf-8') def read_main_log_debug(self) -> str: """Read debug log file.""" - return self.session.main_log_debug.read_text() + return self.session.main_log_debug.read_text(encoding='utf-8') class OutputFile(): diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 78833cc4ab..a6b2ccdd94 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -495,7 +495,7 @@ def write_metadata(products, write_ncl=False): output_filename = os.path.join(output_dir, 'metadata.yml') output_files.append(output_filename) - with open(output_filename, 'w') as file: + with open(output_filename, 'w', encoding='utf-8') as file: yaml.safe_dump(metadata, file) if write_ncl: output_files.append(_write_ncl_metadata(output_dir, metadata)) diff --git a/setup.py b/setup.py index 092379b6ba..84379ea008 100755 --- a/setup.py +++ b/setup.py @@ -183,7 +183,7 @@ def run(self): def read_authors(filename): """Read the list of authors from .zenodo.json file.""" - with Path(filename).open() as file: + with Path(filename).open(encoding='utf-8') as file: info = json.load(file) authors = [] for author in info['creators']: @@ -194,7 +194,7 @@ def read_authors(filename): def read_description(filename): """Read the description from .zenodo.json file.""" - with Path(filename).open() as file: + with Path(filename).open(encoding='utf-8') as file: info = json.load(file) return info['description'] @@ -203,7 +203,7 @@ def read_description(filename): name='ESMValCore', author=read_authors('.zenodo.json'), description=read_description('.zenodo.json'), - long_description=Path('README.md').read_text(), + long_description=Path('README.md').read_text(encoding='utf-8'), long_description_content_type='text/markdown', url='https://www.esmvaltool.org', download_url='https://github.com/ESMValGroup/ESMValCore', diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index be20131e58..ab823cb846 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -2066,7 +2066,7 @@ def test_get_path_from_facet(path, description, output, tmp_path): # Create empty dummy file output = output.format(tmp_path=tmp_path) - with open(output, 'w'): + with open(output, 'w', encoding='utf-8'): pass out_path = fix._get_path_from_facet('test_path', description=description) diff --git a/tests/integration/esgf/test_search_download.py b/tests/integration/esgf/test_search_download.py index 29e215be1d..ea34aab897 100644 --- a/tests/integration/esgf/test_search_download.py +++ b/tests/integration/esgf/test_search_download.py @@ -116,7 +116,7 @@ def test_mock_search(variable, mocker): # Skip cases where the raw search results were too large to save. pytest.skip(f"Raw search results in {raw_results} not available.") - with raw_results.open('r') as file: + with raw_results.open('r', encoding='utf-8') as file: search_results = [ FileResult(json=j, context=None) for j in json.load(file) ] diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 88bde98f31..8e2041bcd3 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -1251,7 +1251,7 @@ def simulate_diagnostic_run(diagnostic_task): plot_file = create_test_image('test', cfg) provenance = os.path.join(cfg['run_dir'], 'diagnostic_provenance.yml') os.makedirs(cfg['run_dir']) - with open(provenance, 'w') as file: + with open(provenance, 'w', encoding='utf-8') as file: yaml.safe_dump({diagnostic_file: record, plot_file: record}, file) diagnostic_task._collect_provenance() diff --git a/tests/integration/test_citation.py b/tests/integration/test_citation.py index 3f1075d43f..339cd253a0 100644 --- a/tests/integration/test_citation.py +++ b/tests/integration/test_citation.py @@ -33,7 +33,7 @@ def test_references(tmp_path, monkeypatch): _write_citation_files(filename, provenance) citation_file = tmp_path / 'output_citation.bibtex' - citation = citation_file.read_text() + citation = citation_file.read_text(encoding='utf-8') assert citation == '\n'.join([ESMVALTOOL_PAPER, fake_bibtex]) @@ -84,7 +84,7 @@ def test_cmip6_data_citation(tmp_path, monkeypatch): \tdoi = {{{doi}}}, }} """).lstrip() - assert citation_file.read_text() == '\n'.join( + assert citation_file.read_text(encoding='utf-8') == '\n'.join( [ESMVALTOOL_PAPER, fake_bibtex_entry]) @@ -114,4 +114,4 @@ def test_cmip6_data_citation_url(tmp_path): f"- {CMIP6_URL_STEM}/cmip6?input={fake_url_prefix}", '', ]) - assert citation_url.read_text() == text + assert citation_url.read_text(encoding='utf-8') == text diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index 243d4c6a28..7816508458 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -56,7 +56,7 @@ def arguments(*args): def check(result_file): """Check the results.""" - result = yaml.safe_load(result_file.read_text()) + result = yaml.safe_load(result_file.read_text(encoding='utf-8')) required_keys = { 'input_files', @@ -80,7 +80,7 @@ def check(result_file): import yaml import shutil - with open("settings.yml", 'r') as file: + with open("settings.yml", 'r', encoding='utf-8') as file: settings = yaml.safe_load(file) shutil.copy("settings.yml", settings["setting_name"]) diff --git a/tests/integration/test_local.py b/tests/integration/test_local.py index 02982177a9..811431b336 100644 --- a/tests/integration/test_local.py +++ b/tests/integration/test_local.py @@ -10,7 +10,9 @@ from esmvalcore.local import LocalFile, _get_output_file, find_files # Load test configuration -with open(os.path.join(os.path.dirname(__file__), 'data_finder.yml')) as file: +with open(os.path.join(os.path.dirname(__file__), + 'data_finder.yml'), + encoding='utf-8') as file: CONFIG = yaml.safe_load(file) @@ -40,7 +42,7 @@ def create_file(filename): if not os.path.exists(dirname): os.makedirs(dirname) - with open(filename, 'a'): + with open(filename, 'a', encoding='utf-8'): pass diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 4eb578150d..94d209ffb5 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -81,7 +81,7 @@ def test_empty_run(tmp_path): Config.get_config_user(path=tmp_path) log_dir = f'{tmp_path}/esmvaltool_output' config_file = f"{tmp_path}/config-user.yml" - with open(config_file, 'r+') as file: + with open(config_file, 'r+', encoding='utf-8') as file: config = yaml.safe_load(file) config['output_dir'] = log_dir yaml.safe_dump(config, file, sort_keys=False) diff --git a/tests/integration/test_task.py b/tests/integration/test_task.py index 3c9801f189..7ea9964585 100644 --- a/tests/integration/test_task.py +++ b/tests/integration/test_task.py @@ -216,7 +216,7 @@ def _get_single_diagnostic_task(tmp_path, diag_script, write_diag=True): diag_run_dir = diag_output_dir / 'run_dir' diag_settings = {'run_dir': diag_run_dir, 'profile_diagnostic': False} if write_diag: - with open(diag_script, "w") as fil: + with open(diag_script, "w", encoding='utf-8') as fil: fil.write("import os\n\nprint(os.getcwd())") task = DiagnosticTask( @@ -273,7 +273,7 @@ def _get_diagnostic_tasks(tmp_path, diagnostic_text, extension): 'exit_on_ncl_warning': False } - with open(diag_script, "w") as fil: + with open(diag_script, "w", encoding='utf-8') as fil: fil.write(diagnostic_text) task = DiagnosticTask( diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 79b9b33149..6372a1c041 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -237,7 +237,7 @@ def test_project_obs4mips_case_correction(tmp_path, monkeypatch, mocker): cfg_dev = { 'obs4mips': project_cfg, } - with cfg_file.open('w') as file: + with cfg_file.open('w', encoding='utf-8') as file: yaml.safe_dump(cfg_dev, file) _config.load_config_developer(cfg_file) @@ -251,7 +251,7 @@ def test_load_config_developer_custom(tmp_path, monkeypatch, mocker): mocker.patch.object(_config, 'read_cmor_tables', autospec=True) cfg_file = tmp_path / 'config-developer.yml' cfg_dev = {'custom': {'cmor_path': '/path/to/tables'}} - with cfg_file.open('w') as file: + with cfg_file.open('w', encoding='utf-8') as file: yaml.safe_dump(cfg_dev, file) _config.load_config_developer(cfg_file) diff --git a/tests/unit/config/test_esgf_pyclient.py b/tests/unit/config/test_esgf_pyclient.py index 6de2f664b0..2c1a028f7b 100644 --- a/tests/unit/config/test_esgf_pyclient.py +++ b/tests/unit/config/test_esgf_pyclient.py @@ -92,7 +92,7 @@ def test_read_config_file(monkeypatch, tmp_path): 'interactive': True }, } - with cfg_file.open('w') as file: + with cfg_file.open('w', encoding='utf-8') as file: yaml.safe_dump(reference, file) cfg = _esgf_pyclient.read_config_file() @@ -113,7 +113,7 @@ def test_read_v25_config_file(monkeypatch, tmp_path): 'url': 'https://some.host/path' }, } - with cfg_file.open('w') as file: + with cfg_file.open('w', encoding='utf-8') as file: yaml.safe_dump(cfg_file_content, file) reference = { diff --git a/tests/unit/documentation/test_changelog.py b/tests/unit/documentation/test_changelog.py index 8a00202e46..3ca052c311 100644 --- a/tests/unit/documentation/test_changelog.py +++ b/tests/unit/documentation/test_changelog.py @@ -7,7 +7,7 @@ def test_duplications_in_changelog(): changelog_path = os.path.join(os.path.dirname(__file__), '../../..', 'doc/changelog.rst') - with open(changelog_path) as changelog: + with open(changelog_path, 'r', encoding='utf-8') as changelog: changelog = changelog.read() # Find all pull requests diff --git a/tests/unit/esgf/test_download.py b/tests/unit/esgf/test_download.py index fd2d50d0f2..3b0030694d 100644 --- a/tests/unit/esgf/test_download.py +++ b/tests/unit/esgf/test_download.py @@ -25,7 +25,7 @@ def test_log_speed(monkeypatch, tmp_path): 200 * megabyte, 16) _download.log_speed('http://otherhost.org/other_file.nc', 4 * megabyte, 1) - with hosts_file.open('r') as file: + with hosts_file.open('r', encoding='utf-8') as file: result = yaml.safe_load(file) expected = { @@ -53,7 +53,7 @@ def test_error(monkeypatch, tmp_path): _download.log_speed('http://somehost.org/some_file.nc', 3 * megabyte, 2) _download.log_error('http://somehost.org/some_file.nc') - with hosts_file.open('r') as file: + with hosts_file.open('r', encoding='utf-8') as file: result = yaml.safe_load(file) expected = { diff --git a/tests/unit/task/test_diagnostic_task.py b/tests/unit/task/test_diagnostic_task.py index 85db69f534..7e6867dd74 100644 --- a/tests/unit/task/test_diagnostic_task.py +++ b/tests/unit/task/test_diagnostic_task.py @@ -20,7 +20,7 @@ def test_write_ncl_settings(tmp_path): } file_name = tmp_path / "settings" write_ncl_settings(settings, file_name) - with open(file_name, 'r') as file: + with open(file_name, 'r', encoding='utf-8') as file: lines = file.readlines() assert 'var_name = "tas"\n' in lines assert 'if (isvar("profile_diagnostic")) then\n' not in lines @@ -32,7 +32,7 @@ def test_write_ncl_settings(tmp_path): } file_name = tmp_path / "settings" write_ncl_settings(settings, file_name) - with open(file_name, 'r') as file: + with open(file_name, 'r', encoding='utf-8') as file: lines = file.readlines() assert 'var_name = "tas"\n' in lines assert 'profile_diagnostic' not in lines diff --git a/tests/unit/task/test_resume_task.py b/tests/unit/task/test_resume_task.py index 8185cd69c2..c5f2693970 100644 --- a/tests/unit/task/test_resume_task.py +++ b/tests/unit/task/test_resume_task.py @@ -16,7 +16,7 @@ def test_run(tmp_path): } } prev_metadata_file = prev_preproc_dir / 'metadata.yml' - with prev_metadata_file.open('w') as file: + with prev_metadata_file.open('w', encoding='utf-8') as file: yaml.safe_dump(prev_metadata, file) output_dir = tmp_path / 'recipe_test_20211001_092100' From 0b07b05298296e8377b417f2d18a355e368e4fb1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 9 Oct 2023 11:54:16 +0100 Subject: [PATCH 22/72] Check type of argument passed to `esmvalcore.cmor.table.read_cmor_tables` (#2217) --- esmvalcore/cmor/table.py | 8 ++++++++ tests/integration/cmor/test_read_cmor_tables.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index aea873a9de..ebce5431ae 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -124,9 +124,17 @@ def read_cmor_tables(cfg_developer: Optional[Path] = None) -> None: ---------- cfg_developer: Path to config-developer.yml file. + + Raises + ------ + TypeError + If `cfg_developer` is not a Path-like object """ if cfg_developer is None: cfg_developer = Path(__file__).parents[1] / 'config-developer.yml' + elif not isinstance(cfg_developer, Path): + raise TypeError("cfg_developer is not a Path-like object, got ", + cfg_developer) mtime = cfg_developer.stat().st_mtime cmor_tables = _read_cmor_tables(cfg_developer, mtime) CMOR_TABLES.clear() diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 4a50e2c76f..32a472b316 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,5 +1,6 @@ from pathlib import Path +import pytest import yaml from esmvalcore.cmor.table import CMOR_TABLES @@ -18,6 +19,14 @@ } +def test_read_cmor_tables_raiser(): + """Test func raiser.""" + cfg_file = {"cow": "moo"} + with pytest.raises(TypeError) as exc: + read_cmor_tables(cfg_file) + assert "cow" in str(exc) + + def test_read_cmor_tables(): """Test that the function `read_cmor_tables` loads the tables correctly.""" table_path = Path(root).parent / 'tables' From be1b790ff56abe3451e0ddb766d35928d45c046e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brei=20Soli=C3=B1o?= Date: Mon, 9 Oct 2023 13:38:16 +0200 Subject: [PATCH 23/72] Dynamic HTML output for monitoring (#2062) Co-authored-by: Klaus Zimmermann --- esmvalcore/experimental/recipe_output.py | 41 +++++- .../experimental/templates/RecipeOutput.j2 | 70 +++++++++- .../experimental/templates/TaskOutput.j2 | 49 ++++--- esmvalcore/experimental/templates/head.j2 | 3 + .../templates/recipe_output_page.j2 | 32 +++-- esmvalcore/experimental/templates/scripts.js | 122 ++++++++++++++++++ tests/unit/experimental/test_recipe_output.py | 52 ++++++++ 7 files changed, 339 insertions(+), 30 deletions(-) create mode 100644 esmvalcore/experimental/templates/scripts.js diff --git a/esmvalcore/experimental/recipe_output.py b/esmvalcore/experimental/recipe_output.py index 69f63765a2..9765d15c77 100644 --- a/esmvalcore/experimental/recipe_output.py +++ b/esmvalcore/experimental/recipe_output.py @@ -2,7 +2,7 @@ import base64 import logging import os.path -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Optional, Tuple, Type @@ -123,6 +123,13 @@ class RecipeOutput(Mapping): The session used to run the recipe. """ + FILTER_ATTRS: list = [ + "realms", + "plot_type", # Used by several diagnostics + "plot_types", + "long_names", + ] + def __init__(self, task_output: dict, session=None, info=None): self._raw_task_output = task_output self._task_output = {} @@ -141,6 +148,7 @@ def __init__(self, task_output: dict, session=None, info=None): diagnostics[name].append(task) # Create diagnostic output + filters: dict = {} for name, tasks in diagnostics.items(): diagnostic_info = info.data['diagnostics'][name] self.diagnostics[name] = DiagnosticOutput( @@ -150,6 +158,36 @@ def __init__(self, task_output: dict, session=None, info=None): description=diagnostic_info.get('description'), ) + # Add data to filters + for task in tasks: + for file in task.files: + RecipeOutput._add_to_filters(filters, file.attributes) + + # Sort at the end because sets are unordered + self.filters = RecipeOutput._sort_filters(filters) + + @classmethod + def _add_to_filters(cls, filters, attributes): + """Add valid values to the HTML output filters.""" + for attr in RecipeOutput.FILTER_ATTRS: + if attr not in attributes: + continue + values = attributes[attr] + # `set()` to avoid duplicates + attr_list = filters.get(attr, set()) + if (isinstance(values, str) or not isinstance(values, Sequence)): + attr_list.add(values) + else: + attr_list.update(values) + filters[attr] = attr_list + + @classmethod + def _sort_filters(cls, filters): + """Sort the HTML output filters.""" + for _filter, _attrs in filters.items(): + filters[_filter] = sorted(_attrs) + return filters + def __repr__(self): """Return canonical string representation.""" string = '\n'.join(repr(item) for item in self._task_output.values()) @@ -218,6 +256,7 @@ def render(self, template=None): diagnostics=self.diagnostics.values(), session=self.session, info=self.info, + filters=self.filters, relpath=os.path.relpath, ) diff --git a/esmvalcore/experimental/templates/RecipeOutput.j2 b/esmvalcore/experimental/templates/RecipeOutput.j2 index c502c39a33..cde84362f9 100644 --- a/esmvalcore/experimental/templates/RecipeOutput.j2 +++ b/esmvalcore/experimental/templates/RecipeOutput.j2 @@ -1,12 +1,72 @@ + + + + +
{% for diagnostic in diagnostics %} -

{{ diagnostic.title }}

-

{{ diagnostic.description }}

+
+

{{ diagnostic.title }}

+

{{ diagnostic.description }}

- {% for task in diagnostic.task_output %} + {% set diagnostic_loop = loop %} + {% for task in diagnostic.task_output %} - {% include 'TaskOutput.j2' %} + {% include 'TaskOutput.j2' %} - {% endfor %} + {% endfor %} +
{% endfor %} +
diff --git a/esmvalcore/experimental/templates/TaskOutput.j2 b/esmvalcore/experimental/templates/TaskOutput.j2 index 48d259ea7c..43753418b2 100644 --- a/esmvalcore/experimental/templates/TaskOutput.j2 +++ b/esmvalcore/experimental/templates/TaskOutput.j2 @@ -2,11 +2,23 @@ {% for file in task.image_files %} -
+
+
- {{ file.caption }} + {{ file.caption }} -
+
{{ file.caption }}

@@ -16,21 +28,28 @@ provenance
+
{% endfor %} -

Data files

+{% if task.data_files|length > 0 %} +

Data files

- + + +{% endif %} diff --git a/esmvalcore/experimental/templates/head.j2 b/esmvalcore/experimental/templates/head.j2 index e6ec19d5ee..0306620f8a 100644 --- a/esmvalcore/experimental/templates/head.j2 +++ b/esmvalcore/experimental/templates/head.j2 @@ -2,6 +2,9 @@ {{ title }} + + +