Support generating multiple packages from packaging_tags

qgreenland/cli/run.py

@@ -73,9 +73,8 @@ def run(
     #     https://github.com/click-contrib/sphinx-click/issues/86#issuecomment-991196764
     from qgreenland.util.config.config import get_config, init_config
     from qgreenland.util.luigi.tasks.pipeline import (
-        LayerPipelines,
-        QGreenlandAll,
-        QGreenlandNoZip,
+        QGreenlandPackages,
+        QGreenlandPackagesNoZip,
     )
 
     if force_package_zip and force_no_package_zip:
@@ -92,11 +91,15 @@ def run(
 
     if fetch_only:
         # Don't do anything except fetch the input asset for each layer.
-        tasks = [LayerPipelines(fetch_only=fetch_only)]
+        # TODO: How to keep "fetch all" functionality? Bring back LayerPipelines solely
+        # for this? Or make the user pass a specific package, or specific a magic word
+        # like "__all__"? ¯\_(ツ)_/¯
+        # tasks = [LayerPipelines(fetch_only=fetch_only)]
+        ...
     elif skip_zip:
-        tasks = [QGreenlandNoZip()]
+        tasks = [QGreenlandPackagesNoZip()]
     else:
-        tasks = [QGreenlandAll()]
+        tasks = [QGreenlandPackages()]
 
     print(f"Running tasks: {str(tasks)}")
     print()

qgreenland/config/helpers/layers/continental_shelf.py

@@ -77,7 +77,7 @@ def make_layers() -> list[Layer]:
             id=f"continental_shelf_{key}",
             title=params["title"],
             description=params["description"],
-            tags=[],
+            packaging_tags=["core"],
             input=LayerInput(
                 dataset=dataset,
                 asset=dataset.assets[key],

qgreenland/config/helpers/layers/esa_cci_surface_elev.py

@@ -43,7 +43,7 @@ def surface_elevation_layer(
         id=f"surface_elevation_change_{variable.lower()}_{start_year}_{end_year}",
         title=f"Surface elevation change {start_year}-{end_year}",
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/geological_map.py

@@ -39,7 +39,7 @@ def make_layer(*, layer_id: str, layer_params: dict) -> Layer:
         id=layer_id,
         title=layer_params["title"],
         description=layer_params["description"],
-        tags=[],
+        packaging_tags=["core"],
         style=layer_params["style"],
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/gravity_anomalies_geoid.py

@@ -23,7 +23,7 @@ def _make_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/lonlat.py

@@ -45,7 +45,7 @@ def _make_lonlat_layer(
         id=asset.id,
         title=f"{title_prefix} lines ({deg} degree)",
         description=(f"Lines of {title_prefix.lower()} in {deg}-degree resolution."),
-        tags=["reference"],
+        packaging_tags=["core"],
         style="lonlat",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/nunagis_protected_areas.py

@@ -152,7 +152,7 @@ def _make_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=nunagis_protected_areas,

qgreenland/config/helpers/layers/racmo.py

@@ -19,7 +19,7 @@ def _make_racmo_wind_vectors() -> Layer:
             """Averaged annual mean wind direction in meters per second from
             RACMO2.3p2 for the period 1958-2019."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="racmo_wind_vectors",
         input=LayerInput(
             dataset=dataset,
@@ -46,7 +46,7 @@ def _make_racmo_wind_speed() -> Layer:
             """Averaged annual mean wind speed in meters per second from RACMO2.3p2
             for the period 1958-2019."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="racmo_wind_speed",
         input=LayerInput(
             dataset=dataset,
@@ -154,7 +154,7 @@ def _make_masked_racmo_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,
@@ -267,7 +267,7 @@ def make_racmo_supplemental_layers() -> list[Layer]:
                 id=layer_id,
                 title=params["title"],
                 description=params["description"],
-                tags=[],
+                packaging_tags=["core"],
                 style=params["style"],
                 input=LayerInput(
                     dataset=dataset,

qgreenland/config/helpers/layers/sea_ice_age.py

@@ -38,7 +38,7 @@ def sea_ice_age_layer(year: int, age_type: AgeType) -> Layer:
             of {age_type} extent chosen based on NSDIC's Sea Ice Index 5-day
             average."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="sea_ice_age",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/streams_outlets_basins.py

@@ -72,7 +72,7 @@
         id=layer_id,
         title=layer_id.replace("_", " ").capitalize(),
         description=params["description"],
-        tags=[],
+        packaging_tags=["core"],
         style=layer_id.replace("_filled", ""),
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/territorial_waters.py

@@ -73,7 +73,7 @@ def make_layers() -> list[Layer]:
             id=key,
             title=params["title"],
             description=params["description"],
-            tags=[],
+            packaging_tags=["core"],
             input=LayerInput(
                 dataset=dataset,
                 asset=dataset.assets["only"],

qgreenland/config/helpers/layers/wmm.py

@@ -24,8 +24,7 @@ def make_boz_layer(*, year: int) -> Layer:
 BoZ, where caution must be exercised while using a compass. Compass accuracy
 may be degraded in this region.
 """,
-        tags=["wmm"],
-        in_package=True,
+        packaging_tags=["core"],
         show=False,
         style="blackout_zones",
         input=LayerInput(
@@ -256,7 +255,7 @@ def make_wmm_variable_layer(
         description=variable_config["description"],
         # We keep the main field declination layers (`d`) in the core
         # package. All other variables will only be available from the plugin.
-        in_package=True if variable == "d" else False,
+        packaging_tags=["core"] if variable == "d" else [],
         style="wmm_contours",
         input=LayerInput(
             dataset=wmm.wmm,

qgreenland/config/helpers/layers/woa.py

@@ -78,7 +78,7 @@ def make_layer(*, dataset, depth, season, variable, units) -> Layer:
             remotely sensed observations, also require a more recent
             climatology."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style=f"seawater_{variable}",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/layers/Basemaps/land_ocean_shape.py

@@ -14,7 +14,7 @@ def make_land_ocean_layer(layer_id: str) -> Layer:
         id=layer_id,
         title=layer_id.capitalize(),
         description=(f"""Polygons representing the {layer_id}."""),
-        tags=[],
+        packaging_tags=["foo"] if layer_id == "land" else ["core", "foo"],
         style=layer_id,
         input=LayerInput(
             dataset=layer_params[layer_id],

qgreenland/config/layers/Reference/arctic_circle.py

@@ -11,7 +11,7 @@
         not set on the summer solstice, and does not rise on the winter
         solstice."""
     ),
-    tags=[],
+    packaging_tags=["core"],
     style="arctic_circle",
     input=LayerInput(
         dataset=dataset,

qgreenland/models/config/layer.py

@@ -36,11 +36,17 @@ class Layer(QgrBaseModel):
     description: str = Field(..., min_length=1)
     """Descriptive text shown as hover-text in the QGIS Layer Panel."""
 
+    # Temporary keeping these around for testing before config is fully migrated
+    in_package: bool = True
     tags: list[str] = []
-    """Additional categories that describe this data."""
 
-    in_package: bool = True
-    """Is this layer in the final QGreenland zip file?"""
+    packaging_tags: list[str] = ["core"]
+    """Which packages does this layer belong in? Layer will be omitted if empty.
+
+    Temporarily defaults to ["core"] to help us test more quickly.
+
+    TODO: Validate len > 0.
+    """
 
     show: bool = False
     """Is this layer initially "checked" or visible in QGIS?"""

qgreenland/test/conftest.py

@@ -27,8 +27,7 @@
     "id": "example_online",
     "title": "Example online",
     "description": "Example layer description.",
-    "tags": ["foo", "bar", "baz"],
-    "in_package": True,
+    "packaging_tags": ["foo", "bar"],
     "input": {
         "dataset": {
             "id": "baz",
@@ -48,8 +47,7 @@
     "id": "example_raster",
     "title": "Example raster",
     "description": "Example layer description.",
-    "tags": ["foo", "bar", "baz"],
-    "in_package": True,
+    "packaging_tags": ["bar", "baz"],
     "input": {
         "dataset": {
             "id": "example_dataset",

qgreenland/test/constants.py

@@ -9,5 +9,6 @@
 TEST_CONFIG_W_MISSING_SETTINGS_DIR = TEST_DATA_DIR / "config_with_missing_settings"
 TEST_CONFIG_W_STRING_ORDER_VALUES = TEST_DATA_DIR / "config_with_string_order_values"
 
+# TODO: Handle new package-specific compile dirs
 MOCK_COMPILE_PACKAGE_DIR = TEST_DATA_DIR / "compile"
 MOCK_RELEASE_LAYERS_DIR = TEST_DATA_DIR / "release" / "layers"

qgreenland/test/util/config/test_config_export.py

@@ -1,96 +1,14 @@
 import csv
-import json
 import tempfile
 from pathlib import Path
 from unittest.mock import patch
 
-from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR, MOCK_RELEASE_LAYERS_DIR
-from qgreenland.util.config.export import export_config_csv, export_config_manifest
+from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR
+from qgreenland.util.config.export import export_config_csv
 
 
 @patch(
-    "qgreenland.util.layer.RELEASE_LAYERS_DIR",
-    new=MOCK_RELEASE_LAYERS_DIR,
-)
-def test_export_config_manifest(full_cfg):
-    common = {
-        "description": "Example layer description.",
-        # TODO: Generate this with imported function? This should be tested
-        # by itself elsewhere, so there's no need to test the expected output
-        # here too.
-        "layer_details": """Example layer description.
-
-=== Original Data Source ===
-Example Dataset
-
-Example abstract.
-
-Citation:
-NSIDC 2020
-
-Citation URL:
-https://nsidc.org""",
-        "tags": ["foo", "bar", "baz"],
-        "hierarchy": ["Group", "Subgroup"],
-    }
-    with tempfile.NamedTemporaryFile("r") as tf:
-        export_config_manifest(
-            full_cfg,
-            output_path=Path(tf.name),
-        )
-
-        actual = json.load(tf)
-
-    assert type(actual["qgr_version"]) is str
-    assert len(actual["qgr_version"]) >= 6
-    del actual["qgr_version"]
-
-    # For now, do not include online layers in the layer manifest. The
-    # `QGreenland Custom` QGIS Plugin does not currently support online
-    # layers. Once online layers are supported in the plugin, this commented out
-    # `online_asset` can be re-added:
-    # online_asset = {
-    #     'type': 'online',
-    #     **full_cfg.layers['example_online'].input.asset.dict(
-    #         include={'provider', 'url'},
-    #     ),
-    # }
-    expected = {
-        "version": "v0.1.0",
-        "layers": [
-            # {
-            #     'id': 'example_online',
-            #     'title': 'Example online',
-            #     'assets': [online_asset],
-            #     **common,
-            # },
-            {
-                "id": "example_raster",
-                "title": "Example raster",
-                "assets": [
-                    {
-                        "checksum": "a9a103f208179726038fa7178747a0a1",
-                        "file": "example.tif",
-                        "size_bytes": 287,
-                        "type": "data",
-                    },
-                    {
-                        "checksum": "22b427acc6e4ebf57052115fdd5ac450",
-                        "file": "example.tif.aux.xml",
-                        "size_bytes": 332,
-                        "type": "ancillary",
-                    },
-                ],
-                **common,
-            },
-        ],
-    }
-
-    assert actual == expected
-
-
-@patch(
-    "qgreenland.util.layer.COMPILE_PACKAGE_DIR",
+    "qgreenland.util.misc.COMPILE_PACKAGE_DIR",
     new=MOCK_COMPILE_PACKAGE_DIR,
 )
 def test_export_config_csv(full_cfg):
@@ -109,6 +27,7 @@ def test_export_config_csv(full_cfg):
         export_config_csv(
             full_cfg,
             output_path=Path(tf.name),
+            package_name="bar",
         )
 
         actual = list(csv.DictReader(tf))

qgreenland/test/util/test_misc.py

@@ -3,8 +3,8 @@
 
 
 def test_layer_compile_dir(raster_layer_node):
-    expected = COMPILE_PACKAGE_DIR / "Group" / "Subgroup" / "Example raster"
-    actual = layer_util.get_layer_compile_dir(raster_layer_node)
+    expected = COMPILE_PACKAGE_DIR / "foo" / "Group" / "Subgroup" / "Example raster"
+    actual = layer_util.get_layer_compile_dir(raster_layer_node, package_name="foo")
 
     assert expected == actual