Merge remote-tracking branch 'origin/main' into nested_config

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as
 contributors and maintainers pledge to making participation in our project and
 our community a harassment-free experience for everyone, regardless of age, body
 size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
+level of experience, education, socioeconomic status, nationality, personal
 appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards

doc/api/esmvalcore.dataset.rst

@@ -1,5 +1,5 @@
 Dataset
-=======
+========
 
 .. automodule:: esmvalcore.dataset
     :no-show-inheritance:

doc/api/esmvalcore.experimental.recipe.rst

@@ -1,7 +1,7 @@
 .. _api_recipe:
 
 Recipes
-=======
+========
 
 This section describes the :py:mod:`~esmvalcore.experimental.recipe` submodule of the API (:py:mod:`esmvalcore.experimental`).
 

doc/changelog.rst

@@ -1024,7 +1024,7 @@ Highlights
 ~~~~~~~~~~
 
 - ESMValCore now has the ability to automatically download missing data from ESGF. For details, see :ref:`Data Retrieval<data-retrieval>`.
-- ESMValCore now also can resume an earlier run. This is useful to re-use expensive preprocessor results. For details, see :ref:`Running<running>`.
+- ESMValCore now also can resume an earlier run. This is useful to reuse expensive preprocessor results. For details, see :ref:`Running<running>`.
 
 This release includes
 

doc/gensidebar.py

@@ -38,7 +38,7 @@ def _toctree():
     def _endl():
         lines.append("")
 
-    def _write(project, desc, link, mapping=conf['intersphinx_mapping']):
+    def _write(project, desc, link, mapping=conf["intersphinx_mapping"]):
         if project != conf_api:
             if do_gen:
                 args = desc, mapping[project][0], link

esmvalcore/__init__.py

@@ -1,4 +1,5 @@
 """ESMValTool core package."""
+
 import logging
 import os
 
@@ -8,9 +9,9 @@
 logger.addHandler(logging.NullHandler())
 
 __all__ = [
-    '__version__',
-    'cmor',
-    'preprocessor',
+    "__version__",
+    "cmor",
+    "preprocessor",
 ]
 
 

esmvalcore/_citation.py

@@ -1,4 +1,5 @@
 """Citation module."""
+
 import logging
 import os
 import re
@@ -11,7 +12,7 @@
 
 logger = logging.getLogger(__name__)
 
-CMIP6_URL_STEM = 'https://cera-www.dkrz.de/WDCC/ui/cerasearch'
+CMIP6_URL_STEM = "https://cera-www.dkrz.de/WDCC/ui/cerasearch"
 
 # The technical overview paper should always be cited
 ESMVALTOOL_PAPER = (
@@ -26,15 +27,16 @@
     "\tpages = {1179--1199},\n"
     "\tauthor = {Mattia Righi and Bouwe Andela and Veronika Eyring "
     "and Axel Lauer and Valeriu Predoi and Manuel Schlund "
-    "and Javier Vegas-Regidor and Lisa Bock and Bj\"{o}rn Br\"{o}tz "
+    'and Javier Vegas-Regidor and Lisa Bock and Bj"{o}rn Br"{o}tz '
     "and Lee de Mora and Faruk Diblen and Laura Dreyer "
     "and Niels Drost and Paul Earnshaw and Birgit Hassler "
     "and Nikolay Koldunov and Bill Little and Saskia Loosveldt Tomas "
     "and Klaus Zimmermann},\n"
     "\ttitle = {Earth System Model Evaluation Tool (ESMValTool) v2.0 "
     "-- technical overview},\n"
     "\tjournal = {Geoscientific Model Development}\n"
-    "}\n")
+    "}\n"
+)
 
 
 def _write_citation_files(filename, provenance):
@@ -56,22 +58,22 @@ def _write_citation_files(filename, provenance):
 
     for item in provenance.records:
         # get cmip6 data citation info
-        cmip6_data = 'CMIP6' in item.get_attribute('attribute:mip_era')
+        cmip6_data = "CMIP6" in item.get_attribute("attribute:mip_era")
         if cmip6_data:
             url_prefix = _make_url_prefix(item.attributes)
             cmip6_info_urls.add(_make_info_url(url_prefix))
             cmip6_json_urls.add(_make_json_url(url_prefix))
 
         # get other citation info
-        references = item.get_attribute('attribute:references')
+        references = item.get_attribute("attribute:references")
         if not references:
             # ESMValTool CMORization scripts use 'reference' (without final s)
-            references = item.get_attribute('attribute:reference')
+            references = item.get_attribute("attribute:reference")
         if references:
-            if item.identifier.namespace.prefix == 'recipe':
+            if item.identifier.namespace.prefix == "recipe":
                 # get recipe citation tags
                 tags.update(references)
-            elif item.get_attribute('attribute:script_file'):
+            elif item.get_attribute("attribute:script_file"):
                 # get diagnostics citation tags
                 tags.update(references)
             elif not cmip6_data:
@@ -101,9 +103,10 @@ def _save_citation_bibtex(product_name, tags, json_urls):
             entries.add(cmip_citation)
     citation_entries.extend(sorted(entries))
 
-    with open(f'{product_name}_citation.bibtex',
-              'w', encoding='utf-8') as file:
-        file.write('\n'.join(citation_entries))
+    with open(
+        f"{product_name}_citation.bibtex", "w", encoding="utf-8"
+    ) as file:
+        file.write("\n".join(citation_entries))
 
 
 def _save_citation_info_txt(product_name, info_urls, other_info):
@@ -114,22 +117,26 @@ def _save_citation_info_txt(product_name, info_urls, other_info):
         lines.append(
             "Follow the links below to find more information about CMIP6 data:"
         )
-        lines.extend(f'- {url}' for url in sorted(info_urls))
+        lines.extend(f"- {url}" for url in sorted(info_urls))
 
     # Save any references from the 'references' and 'reference' NetCDF global
     # attributes.
     if other_info:
         if lines:
-            lines.append('')
-        lines.append("Additional data citation information was found, for "
-                     "which no entry is available in the bibtex file:")
-        lines.extend('- ' + str(t).replace('\n', ' ')
-                     for t in sorted(other_info))
+            lines.append("")
+        lines.append(
+            "Additional data citation information was found, for "
+            "which no entry is available in the bibtex file:"
+        )
+        lines.extend(
+            "- " + str(t).replace("\n", " ") for t in sorted(other_info)
+        )
 
     if lines:
-        with open(f'{product_name}_data_citation_info.txt',
-                  'w', encoding='utf-8') as file:
-            file.write('\n'.join(lines) + '\n')
+        with open(
+            f"{product_name}_data_citation_info.txt", "w", encoding="utf-8"
+        ) as file:
+            file.write("\n".join(lines) + "\n")
 
 
 def _extract_tags(tags):
@@ -139,46 +146,48 @@ def _extract_tags(tags):
     For example, a single entry in the list `tags` could be the string
     "['acknow_project', 'acknow_author']".
     """
-    pattern = re.compile(r'[\w-]+')
+    pattern = re.compile(r"[\w-]+")
     return set(pattern.findall(str(tags)))
 
 
 def _get_response(url):
     """Return information from CMIP6 Data Citation service in json format."""
     json_data = None
-    if url.lower().startswith('https'):
+    if url.lower().startswith("https"):
         try:
             response = requests.get(url, timeout=5)
             if response.status_code == 200:
                 json_data = response.json()
             else:
-                logger.warning('Error in the CMIP6 citation link: %s', url)
+                logger.warning("Error in the CMIP6 citation link: %s", url)
         except IOError:
-            logger.info('No network connection, '
-                        'unable to retrieve CMIP6 citation information')
+            logger.info(
+                "No network connection, "
+                "unable to retrieve CMIP6 citation information"
+            )
     return json_data
 
 
 def _json_to_bibtex(data):
     """Make a bibtex entry from CMIP6 Data Citation json data."""
-    url = 'url not found'
-    title = data.get('titles', ['title not found'])[0]
-    publisher = data.get('publisher', 'publisher not found')
-    year = data.get('publicationYear', 'publicationYear not found')
-    authors = 'creators not found'
-    doi = 'doi not found'
-
-    if 'creators' in data:
+    url = "url not found"
+    title = data.get("titles", ["title not found"])[0]
+    publisher = data.get("publisher", "publisher not found")
+    year = data.get("publicationYear", "publicationYear not found")
+    authors = "creators not found"
+    doi = "doi not found"
+
+    if "creators" in data:
         author_list = [
-            item.get('creatorName', '') for item in data['creators']
+            item.get("creatorName", "") for item in data["creators"]
         ]
-        authors = ' and '.join(author_list)
+        authors = " and ".join(author_list)
         if not authors:
-            authors = 'creators not found'
+            authors = "creators not found"
 
-    if 'identifier' in data:
-        doi = data['identifier'].get('id', 'doi not found')
-        url = f'https://doi.org/{doi}'
+    if "identifier" in data:
+        doi = data["identifier"].get("id", "doi not found")
+        url = f"https://doi.org/{doi}"
 
     bibtex_entry = textwrap.dedent(f"""
         @misc{{{url},
@@ -196,14 +205,16 @@ def _json_to_bibtex(data):
 @lru_cache(maxsize=1024)
 def _collect_bibtex_citation(tag):
     """Collect information from bibtex files."""
-    bibtex_file = DIAGNOSTICS.references / f'{tag}.bibtex'
+    bibtex_file = DIAGNOSTICS.references / f"{tag}.bibtex"
     if bibtex_file.is_file():
-        entry = bibtex_file.read_text(encoding='utf-8')
+        entry = bibtex_file.read_text(encoding="utf-8")
     else:
-        entry = ''
+        entry = ""
         logger.warning(
             "The reference file %s does not exist, citation information "
-            "incomplete.", bibtex_file)
+            "incomplete.",
+            bibtex_file,
+        )
     return entry
 
 
@@ -214,34 +225,34 @@ def _collect_cmip_citation(json_url):
     if json_data:
         bibtex_entry = _json_to_bibtex(json_data)
     else:
-        bibtex_entry = ''
+        bibtex_entry = ""
     return bibtex_entry
 
 
 def _make_url_prefix(attribute):
     """Make url prefix based on CMIP6 Data Citation Service."""
     # the order of keys is important
     localpart = {
-        'mip_era': '',
-        'activity_id': '',
-        'institution_id': '',
-        'source_id': '',
-        'experiment_id': '',
+        "mip_era": "",
+        "activity_id": "",
+        "institution_id": "",
+        "source_id": "",
+        "experiment_id": "",
     }
     for key, value in attribute:
         if key.localpart in localpart:
             localpart[key.localpart] = value
-    url_prefix = '.'.join(localpart.values())
+    url_prefix = ".".join(localpart.values())
     return url_prefix
 
 
 def _make_json_url(url_prefix):
     """Make json url based on CMIP6 Data Citation Service."""
-    json_url = f'{CMIP6_URL_STEM}/cerarest/exportcmip6?input={url_prefix}'
+    json_url = f"{CMIP6_URL_STEM}/cerarest/exportcmip6?input={url_prefix}"
     return json_url
 
 
 def _make_info_url(url_prefix):
     """Make info url based on CMIP6 Data Citation Service."""
-    info_url = f'{CMIP6_URL_STEM}/cmip6?input={url_prefix}'
+    info_url = f"{CMIP6_URL_STEM}/cmip6?input={url_prefix}"
     return info_url