Merge branch 'main' into fix-overwriting-facets-from-cmor-table

.circleci/config.yml

@@ -159,7 +159,7 @@ jobs:
     working_directory: /esmvaltool
     docker:
       - image: condaforge/mambaforge
-    resource_class: small
+    resource_class: medium
     steps:
       - run:
           command: |
@@ -182,7 +182,7 @@ jobs:
     # Test building documentation
     docker:
       - image: condaforge/mambaforge
-    resource_class: small
+    resource_class: medium
     steps:
       - checkout
       - run:

.github/workflows/build-and-deploy-on-pypi.yml

@@ -12,12 +12,18 @@ jobs:
   build-n-publish:
     name: Build and publish ESMValCore on PyPi
     runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/ESMValCore/
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set up Python 3.11
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.11"
       - name: Install pep517
@@ -34,14 +40,10 @@ jobs:
           --binary
           --out-dir dist/
           .
-      #- name: Publish distribution 📦 to Test PyPI
-      #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
-      #    password: ${{ secrets.test_pypi_password }}
-      #    repository_url: https://test.pypi.org/legacy/
+      # - name: Publish distribution to Test PyPI
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     repository-url: https://test.pypi.org/legacy/
       - name: Publish distribution 📦 to PyPI
         if: startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.pypi_password }}

.github/workflows/install-from-condalock-file.yml

@@ -46,7 +46,6 @@ jobs:
       - run: which python
       - run: python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
       - run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
-      - run: conda install pip
       - run: which python
       - run: pip --version
       - run: pip install -e .[develop]

CITATION.cff

@@ -201,11 +201,11 @@ authors:
     given-names: Joerg
 
 cff-version: 1.2.0
-date-released: 2023-11-01
+date-released: 2023-12-19
 doi: "10.5281/zenodo.3387139"
 license: "Apache-2.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/ESMValGroup/ESMValCore/"
 title: ESMValCore
-version: "v2.10.0rc1"
+version: "v2.10.0"
 ...

README.md

@@ -7,7 +7,7 @@
 [![codecov](https://codecov.io/gh/ESMValGroup/ESMValCore/branch/main/graph/badge.svg?token=wQnDzguwq6)](https://codecov.io/gh/ESMValGroup/ESMValCore)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/5d496dea9ef64ec68e448a6df5a65783)](https://www.codacy.com/gh/ESMValGroup/ESMValCore?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValCore&utm_campaign=Badge_Grade)
 [![Docker Build Status](https://img.shields.io/docker/cloud/build/esmvalgroup/esmvalcore)](https://hub.docker.com/r/esmvalgroup/esmvalcore/)
-[![Anaconda-Server Badge](https://img.shields.io/badge/Anaconda.org-2.9.0-blue.svg)](https://anaconda.org/conda-forge/esmvalcore)
+[![Anaconda-Server Badge](https://img.shields.io/conda/vn/conda-forge/ESMValCore?color=blue&label=conda-forge&logo=conda-forge&logoColor=white)](https://anaconda.org/conda-forge/esmvalcore)
 [![Github Actions Test](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml)
 
 ![esmvaltoollogo](https://raw.githubusercontent.com/ESMValGroup/ESMValCore/main/doc/figures/ESMValTool-logo-2.png)

doc/conf.py

@@ -60,6 +60,7 @@
     'nbsphinx',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
+    'sphinx.ext.extlinks',
     'sphinx.ext.intersphinx',
     'sphinx.ext.todo',
     'sphinx.ext.coverage',
@@ -450,6 +451,36 @@
     'scipy': ('https://docs.scipy.org/doc/scipy/', None),
 }
 
+# -- Extlinks extension -------------------------------------------------------
+# See https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html
+
+extlinks = {
+    "discussion": (
+        "https://github.com/ESMValGroup/ESMValCore/discussions/%s",
+        "Discussion #%s",
+    ),
+    "issue": (
+        "https://github.com/ESMValGroup/ESMValCore/issues/%s",
+        "Issue #%s",
+    ),
+    "pull": (
+        "https://github.com/ESMValGroup/ESMValCore/pull/%s",
+        "Pull request #%s",
+    ),
+    "release": (
+        "https://github.com/ESMValGroup/ESMValCore/releases/tag/%s",
+        "ESMValCore %s",
+    ),
+    "team": (
+        "https://github.com/orgs/ESMValGroup/teams/%s",
+        "@ESMValGroup/%s",
+    ),
+    "user": (
+        "https://github.com/%s",
+        "@%s",
+    ),
+}
+
 # -- Custom Document processing ----------------------------------------------
 
 sys.path.append(os.path.dirname(__file__))

doc/quickstart/configure.rst

@@ -243,8 +243,7 @@ Extensive documentation on setting up Dask Clusters is available
 
   If not all preprocessor functions support lazy data, computational
   performance may be best with the default scheduler.
-  See `issue #674 <https://github.com/ESMValGroup/ESMValCore/issues/674>`_ for
-  progress on making all preprocessor functions lazy.
+  See :issue:`674` for progress on making all preprocessor functions lazy.
 
 **Example configurations**
 
@@ -684,7 +683,8 @@ related to CMOR table settings available:
   extended with variables from the :ref:`custom_cmor_tables` (by default loaded
   from the ``esmvalcore/cmor/tables/custom`` directory) and it is possible to
   use variables with a ``mip`` which is different from the MIP table in which
-  they are defined.
+  they are defined. Note that this option is always enabled for
+  :ref:`derived <Variable derivation>` variables.
 * ``cmor_path``: path to the CMOR table.
   Relative paths are with respect to `esmvalcore/cmor/tables`_.
   Defaults to the value provided in ``cmor_type`` written in lower case.
@@ -699,11 +699,18 @@ Custom CMOR tables
 
 As mentioned in the previous section, the CMOR tables of projects that use
 ``cmor_strict: false`` will be extended with custom CMOR tables.
-By default, these are loaded from `esmvalcore/cmor/tables/custom
+For derived variables (the ones with ``derive: true`` in the recipe), the
+custom CMOR tables will always be considered.
+By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom
 <https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom>`_.
 However, by using the special project ``custom`` in the
 ``config-developer.yml`` file with the option ``cmor_path``, a custom location
-for these custom CMOR tables can be specified:
+for these custom CMOR tables can be specified.
+In this case, the default custom tables are extended with those entries from
+the custom location (in case of duplication, the custom location tables take
+precedence).
+
+Example:
 
 .. code-block:: yaml
 
@@ -743,11 +750,10 @@ Example for the file ``CMOR_asr.dat``:
    !----------------------------------
    !
 
-It is also possible to use a special coordinates file ``CMOR_coordinates.dat``.
-If this is not present in the custom directory, the one from the default
-directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat
-<https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat>`_)
-is used.
+It is also possible to use a special coordinates file ``CMOR_coordinates.dat``,
+which will extend the entries from the default one
+(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat
+<https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat>`_).
 
 
 .. _filterwarnings_config-developer:

doc/recipe/preprocessor.rst

@@ -201,13 +201,15 @@ the ozone 3D field. In this case, a derivation function is provided to
 vertically integrate the ozone and obtain total column ozone for direct
 comparison with the observations.
 
-To contribute a new derived variable, it is also necessary to define a name for
-it and to provide the corresponding CMOR table. This is to guarantee the proper
-metadata definition is attached to the derived data. Such custom CMOR tables
-are collected as part of the `ESMValCore package
-<https://github.com/ESMValGroup/ESMValCore>`_. By default, the variable
-derivation will be applied only if the variable is not already available in the
-input data, but the derivation can be forced by setting the appropriate flag.
+The tool will also look in other ``mip`` tables for the same ``project`` to find
+the definition of derived variables. To contribute a completely new derived
+variable, it is necessary to define a name for it and to provide the
+corresponding CMOR table. This is to guarantee the proper metadata definition
+is attached to the derived data. Such custom CMOR tables are collected as part
+of the `ESMValCore package <https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom>`_.
+By default, the variable derivation will be applied only if the variable is not
+already available in the input data, but the derivation can be forced by
+setting the ``force_derivation`` flag.
 
 .. code-block:: yaml
 
@@ -1219,6 +1221,7 @@ The ``_time.py`` module contains the following preprocessor functions:
 * regrid_time_: Aligns the time axis of each dataset to have common time
   points and calendars.
 * timeseries_filter_: Allows application of a filter to the time-series data.
+* local_solar_time_: Convert cube with UTC time to local solar time.
 
 Statistics functions are applied by default in the order they appear in the
 list. For example, the following example applied to hourly data will retrieve
@@ -1653,6 +1656,55 @@ Examples:
 
 See also :func:`esmvalcore.preprocessor.timeseries_filter`.
 
+.. _local_solar_time:
+
+``local_solar_time``
+--------------------
+
+Many variables in the Earth system show a strong diurnal cycle.
+The reason for that is of course Earth's rotation around its own axis, which
+leads to a diurnal cycle of the incoming solar radiation.
+While UTC time is a very good absolute time measure, it is not really suited to
+analyze diurnal cycles over larger regions.
+For example, diurnal cycles over Russia and the USA are phase-shifted by ~180°
+= 12 hr in UTC time.
+
+This is where the `local solar time (LST)
+<https://en.wikipedia.org/wiki/Solar_time>`__ comes into play:
+For a given location, 12:00 noon LST is defined as the moment when the sun
+reaches its highest point in the sky.
+By using this definition based on the origin of the diurnal cycle (the sun), we
+can directly compare diurnal cycles across the globe.
+LST is mainly determined by the longitude of a location, but due to the
+eccentricity of Earth's orbit, it also depends on the day of year (see
+`equation of time <https://en.wikipedia.org/wiki/Equation_of_time>`__).
+However, this correction is at most ~15 min, which is usually smaller than the
+highest frequency output of CMIP6 models (1 hr) and smaller than the time scale
+for diurnal evolution of meteorological phenomena (which is in the order of
+hours, not minutes).
+Thus, instead, we use the **mean** LST, which solely depends on longitude:
+
+.. math::
+
+  LST = UTC + 12 \cdot \frac{lon}{180°}
+
+where the times are given in hours and `lon` in degrees in the interval [-180,
+180].
+To transform data from UTC to LST, this preprocessor shifts data along the time
+axis based on the longitude.
+
+This preprocessor does not need any additional parameters.
+
+Example:
+
+.. code-block:: yaml
+
+  calculate_local_solar_time:
+    local_solar_time:
+
+See also :func:`esmvalcore.preprocessor.local_solar_time`.
+
+
 .. _area operations:
 
 Area manipulation

environment.yml

@@ -20,7 +20,7 @@ dependencies:
   - geopy
   - humanfriendly
   - importlib_metadata  # required for Python < 3.10
-  - iris >=3.6.0
+  - iris >=3.6.1
   - iris-esmf-regrid >=0.7.0
   - isodate
   - jinja2
@@ -37,7 +37,7 @@ dependencies:
   - psutil
   - py-cordex
   - pybtex
-  - python >=3.9
+  - python >=3.9,<3.12
   - python-stratify >=0.3
   - pyyaml
   - requests

esmvalcore/_recipe/recipe.py

@@ -250,25 +250,42 @@ def _add_to_download_list(dataset):
 
 
 def _schedule_for_download(datasets):
-    """Schedule files for download and show the list of files in the log."""
+    """Schedule files for download."""
     for dataset in datasets:
         _add_to_download_list(dataset)
         for supplementary_ds in dataset.supplementaries:
             _add_to_download_list(supplementary_ds)
 
-        files = list(dataset.files)
-        for supplementary_ds in dataset.supplementaries:
-            files.extend(supplementary_ds.files)
+
+def _log_input_files(datasets: Iterable[Dataset]) -> None:
+    """Show list of files in log (including supplementaries)."""
+    for dataset in datasets:
+        # Only log supplementary variables if present
+        supplementary_files_str = ""
+        if dataset.supplementaries:
+            for sup_ds in dataset.supplementaries:
+                supplementary_files_str += (
+                    f"\nwith files for supplementary variable "
+                    f"{sup_ds['short_name']}:\n{_get_files_str(sup_ds)}"
+                )
 
         logger.debug(
-            "Using input files for variable %s of dataset %s:\n%s",
+            "Using input files for variable %s of dataset %s:\n%s%s",
             dataset.facets['short_name'],
-            dataset.facets['alias'].replace('_', ' '),
-            '\n'.join(f'{f} (will be downloaded)' if not f.exists() else str(f)
-                      for f in files),
+            dataset.facets['alias'].replace('_', ' '),  # type: ignore
+            _get_files_str(dataset),
+            supplementary_files_str
         )
 
 
+def _get_files_str(dataset: Dataset) -> str:
+    """Get nice string representation of all files of a dataset."""
+    return '\n'.join(
+        f'  {f}' if f.exists()  # type: ignore
+        else f'  {f} (will be downloaded)' for f in dataset.files
+    )
+
+
 def _check_input_files(input_datasets: Iterable[Dataset]) -> set[str]:
     """Check that the required input files are available."""
     missing = set()
@@ -517,6 +534,7 @@ def _get_preprocessor_products(
         _set_version(dataset, input_datasets)
         USED_DATASETS.append(dataset)
         _schedule_for_download(input_datasets)
+        _log_input_files(input_datasets)
         logger.info("Found input files for %s", dataset.summary(shorten=True))
 
         filename = _get_output_file(