Merge branch 'main' into 199-cordex-xfails

docs/contributing/sources.rst

@@ -8,4 +8,13 @@ Intake-ESM datastore for some climate data on Gadi (e.g. by following :ref:`data
 to be findable and useable by others in the community. Or you're aware of an ACCESS-related climate data product on 
 Gadi that you think should be included in the catalog. Either way, we'd like to hear from you. Please open a 
 catalog data request `here <https://github.com/ACCESS-NRI/access-nri-intake-catalog/issues/new/choose>`_ providing 
-details of the data product to add.
+details of the data product to add.
+
+.. warning:: 
+    If you are providing an existing Intake-ESM datastore to be added to :code:`access-nri-intake-catalog`, the 
+    datastore must be in its final form **before** you make a data request. If a datastore is changed
+    after we have verified that we are able to ingest it, it will break future catalog builds and may be 
+    removed.
+
+    If you need to update a datastore that is already in :code:`access-nri-intake-catalog`, please contact us as
+    described above.

docs/datastores/adding.rst

@@ -19,4 +19,13 @@ we're happy to help you through the process.
 .. note::
    Datastores don't have to have been created by access-nri-intake Builders in order to be added to the 
    catalog. If you have an Intake-ESM datastore (or indeed another type of Intake source) that you think should be in the 
-   catalog, please open a catalog data request.
+   catalog, please open a catalog data request.
+
+.. warning:: 
+    If you are providing an existing Intake-ESM datastore to be added to :code:`access-nri-intake-catalog`, the 
+    datastore must be in its final form **before** you make a data request. If a datastore is changed
+    after we have verified that we are able to ingest it, it will break future catalog builds and may be 
+    removed.
+
+    If you need to update a datastore that is already in :code:`access-nri-intake-catalog`, please contact us as
+    described above.

docs/generate_includes.py

@@ -3,25 +3,46 @@
 
 """ Generate includes for documentation """
 
-import os
+import re
+import warnings
+from pathlib import Path
 
 import yaml
 
+STORAGE_FLAG_REGEXP = r"^/g/data/(?P<proj>[a-z]{1,2}[0-9]{1,2})/.*?$"
+
+
+def storage_includes() -> None:
+    here = Path(__file__).parent.absolute()
+
+    project_list = set()
+    for source_yaml in (here.parent / "config").glob("*.yaml"):
+        print(source_yaml)
+        with open(source_yaml) as fobj:
+            contents = yaml.safe_load(fobj)
+
+        # Loop over the sources in the YAML, extract all storage flags
+        # Will ignore anything that doesn't look like /g/data/<flag>/....
+        try:
+            for source in contents["sources"]:
+                metadata_match = re.match(STORAGE_FLAG_REGEXP, source["metadata_yaml"])
+                if metadata_match:
+                    project_list.add(metadata_match.group("proj"))
+                for data_path in source["path"]:
+                    data_path_match = re.match(STORAGE_FLAG_REGEXP, data_path)
+                    if data_path_match:
+                        project_list.add(data_path_match.group("proj"))
+        except KeyError:
+            warnings.warn(f"Unable to parse config YAML file {source_yaml} - skipping")
+            continue
 
-def storage_includes():
-    here = os.path.abspath(os.path.dirname(__file__))
-    with open(
-        os.path.join(here, "..", "src", "access_nri_intake", "data", "catalog.yaml")
-    ) as fobj:
-        contents = yaml.safe_load(fobj)
-    storage_flags = contents["sources"]["access_nri"]["metadata"]["storage"]
-    project_list = [
-        f"* :code:`{proj.removeprefix('gdata/')}`" for proj in storage_flags.split("+")
-    ]
-    with open("storage_flags.rst", "w") as fobj:
-        fobj.write(f".. code-block::\n\n   {storage_flags}")
     with open("project_list.rst", "w") as fobj:
-        fobj.write("\n".join(project_list) + "\n")
+        [fobj.write(f"* :code:`{proj}`\n") for proj in project_list]
+    storage_string = "+".join([f"gdata/{proj}" for proj in project_list])
+    with open("storage_flags.rst", "w") as fobj:
+        fobj.write(f".. code-block::\n\n   {storage_string}")
+
+    return None
 
 
 if __name__ == "__main__":

docs/management/release.rst

@@ -15,6 +15,13 @@ or may not include an update to the ACCESS-NRI catalog files on Gadi.
 
 #. Enter the new version (vX.X.X) as the tag and release title. Add a brief description of the release.
 
+   .. note::
+
+      It is recommended to attempt a beta release before committing to a major code update.
+      In this case, the version number requires an ordinal after the :code:`b`, e.g., :code:`vYYYY-MM-DDb0`. If the
+      ordinal isn't provided, the GitHub PyPI build action will append one, which breaks the linkage
+      between the PyPI and Conda build actions.
+
 #. Click on "Publish release". This should create the release on GitHub and trigger the workflow that builds and uploads 
    the new version to PyPI and conda
 
@@ -35,9 +42,27 @@ Generating a new catalog version
      $ cd bin
      $ qsub -v version=${RELEASE} build_all.sh
 
+   .. note::
+      Running the build script requires access to an up-to-date checkout of the :code:`access-nri-intake-catalog`
+      repository. The default location for this is :code:`/g/data/xp65/admin/access-nri-intake-catalog`. If you do 
+      not have the ability to update this checkout, you may use a local one; however, you will need to update
+      the :code:`CONFIG_DIR` variable in :code:`bin/build_all.sh` to point at your checkout location.
+
    .. note:: 
       If :code:`version` is not provided, the default used is the current date, in the format :code:`vYYYY-MM-DD`. This should 
       be acceptable in most cases.
     
 #. Updating :code:`access_nri_intake_catalog` is no longer necessary - the new catalog will be available immediately as 
    :code:`intake.cat.access_nri`.
+
+
+New release with new catalog
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In the case of a linked release of a new major :code:`access-nri-intake-catalog` and a new catalog 
+build, the recommened process is:
+
+#. Create a beta release of :code:`access-nri-intake-catalog`;
+#. Use the beta release to build a new catalog;
+#. Iterate over the above steps until the desired result is achieved;
+#. Make a definitive code release.

docs/project_list.rst

@@ -1,9 +1,13 @@
+* :code:`rr3`
+* :code:`rt52`
+* :code:`hq89`
+* :code:`zz63`
 * :code:`al33`
-* :code:`cj50`
-* :code:`dk92`
-* :code:`fs38`
-* :code:`ik11`
 * :code:`oi10`
+* :code:`ik11`
+* :code:`ig45`
+* :code:`fs38`
 * :code:`p73`
-* :code:`rr3`
 * :code:`xp65`
+* :code:`py18`
+* :code:`cj50`

docs/storage_flags.rst

@@ -1,3 +1,3 @@
 .. code-block::
 
-   gdata/al33+gdata/cj50+gdata/dk92+gdata/fs38+gdata/ik11+gdata/oi10+gdata/p73+gdata/rr3+gdata/xp65
+   gdata/rr3+gdata/rt52+gdata/hq89+gdata/zz63+gdata/al33+gdata/oi10+gdata/ik11+gdata/ig45+gdata/fs38+gdata/p73+gdata/xp65+gdata/py18+gdata/cj50