WIP: Add support for building multiple packages, migrate some layers

qgreenland/cli/run.py

@@ -73,9 +73,8 @@ def run(
     #     https://github.com/click-contrib/sphinx-click/issues/86#issuecomment-991196764
     from qgreenland.util.config.config import get_config, init_config
     from qgreenland.util.luigi.tasks.pipeline import (
-        LayerPipelines,
-        QGreenlandAll,
-        QGreenlandNoZip,
+        QGreenlandPackages,
+        QGreenlandPackagesNoZip,
     )
 
     if force_package_zip and force_no_package_zip:
@@ -92,11 +91,15 @@ def run(
 
     if fetch_only:
         # Don't do anything except fetch the input asset for each layer.
-        tasks = [LayerPipelines(fetch_only=fetch_only)]
+        # TODO: How to keep "fetch all" functionality? Bring back LayerPipelines solely
+        # for this? Or make the user pass a specific package, or specific a magic word
+        # like "__all__"? ¯\_(ツ)_/¯
+        # tasks = [LayerPipelines(fetch_only=fetch_only)]
+        ...
     elif skip_zip:
-        tasks = [QGreenlandNoZip()]
+        tasks = [QGreenlandPackagesNoZip()]
     else:
-        tasks = [QGreenlandAll()]
+        tasks = [QGreenlandPackages()]
 
     print(f"Running tasks: {str(tasks)}")
     print()

qgreenland/config/helpers/layers/continental_shelf.py

@@ -77,7 +77,7 @@ def make_layers() -> list[Layer]:
             id=f"continental_shelf_{key}",
             title=params["title"],
             description=params["description"],
-            tags=[],
+            packaging_tags=["core"],
             input=LayerInput(
                 dataset=dataset,
                 asset=dataset.assets[key],

qgreenland/config/helpers/layers/esa_cci_surface_elev.py

@@ -43,7 +43,7 @@ def surface_elevation_layer(
         id=f"surface_elevation_change_{variable.lower()}_{start_year}_{end_year}",
         title=f"Surface elevation change {start_year}-{end_year}",
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/geological_map.py

@@ -39,7 +39,7 @@ def make_layer(*, layer_id: str, layer_params: dict) -> Layer:
         id=layer_id,
         title=layer_params["title"],
         description=layer_params["description"],
-        tags=[],
+        packaging_tags=["core"],
         style=layer_params["style"],
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/gravity_anomalies_geoid.py

@@ -23,7 +23,7 @@ def _make_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/lonlat.py

@@ -45,7 +45,7 @@ def _make_lonlat_layer(
         id=asset.id,
         title=f"{title_prefix} lines ({deg} degree)",
         description=(f"Lines of {title_prefix.lower()} in {deg}-degree resolution."),
-        tags=["reference"],
+        packaging_tags=["core"],
         style="lonlat",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/nunagis_protected_areas.py

@@ -152,7 +152,7 @@ def _make_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=nunagis_protected_areas,

qgreenland/config/helpers/layers/racmo.py

@@ -19,7 +19,7 @@ def _make_racmo_wind_vectors() -> Layer:
             """Averaged annual mean wind direction in meters per second from
             RACMO2.3p2 for the period 1958-2019."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="racmo_wind_vectors",
         input=LayerInput(
             dataset=dataset,
@@ -46,7 +46,7 @@ def _make_racmo_wind_speed() -> Layer:
             """Averaged annual mean wind speed in meters per second from RACMO2.3p2
             for the period 1958-2019."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="racmo_wind_speed",
         input=LayerInput(
             dataset=dataset,
@@ -154,7 +154,7 @@ def _make_masked_racmo_layer(
         id=layer_id,
         title=title,
         description=description,
-        tags=[],
+        packaging_tags=["core"],
         style=style,
         input=LayerInput(
             dataset=dataset,
@@ -267,7 +267,7 @@ def make_racmo_supplemental_layers() -> list[Layer]:
                 id=layer_id,
                 title=params["title"],
                 description=params["description"],
-                tags=[],
+                packaging_tags=["core"],
                 style=params["style"],
                 input=LayerInput(
                     dataset=dataset,

qgreenland/config/helpers/layers/sea_ice_age.py

@@ -38,7 +38,7 @@ def sea_ice_age_layer(year: int, age_type: AgeType) -> Layer:
             of {age_type} extent chosen based on NSDIC's Sea Ice Index 5-day
             average."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style="sea_ice_age",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/streams_outlets_basins.py

@@ -72,7 +72,7 @@
         id=layer_id,
         title=layer_id.replace("_", " ").capitalize(),
         description=params["description"],
-        tags=[],
+        packaging_tags=["core"],
         style=layer_id.replace("_filled", ""),
         input=LayerInput(
             dataset=dataset,

qgreenland/config/helpers/layers/territorial_waters.py

@@ -73,7 +73,7 @@ def make_layers() -> list[Layer]:
             id=key,
             title=params["title"],
             description=params["description"],
-            tags=[],
+            packaging_tags=["core"],
             input=LayerInput(
                 dataset=dataset,
                 asset=dataset.assets["only"],

qgreenland/config/helpers/layers/wmm.py

@@ -24,8 +24,7 @@ def make_boz_layer(*, year: int) -> Layer:
 BoZ, where caution must be exercised while using a compass. Compass accuracy
 may be degraded in this region.
 """,
-        tags=["wmm"],
-        in_package=True,
+        packaging_tags=["core"],
         show=False,
         style="blackout_zones",
         input=LayerInput(
@@ -256,7 +255,7 @@ def make_wmm_variable_layer(
         description=variable_config["description"],
         # We keep the main field declination layers (`d`) in the core
         # package. All other variables will only be available from the plugin.
-        in_package=True if variable == "d" else False,
+        packaging_tags=["core"] if variable == "d" else [],
         style="wmm_contours",
         input=LayerInput(
             dataset=wmm.wmm,

qgreenland/config/helpers/layers/woa.py

@@ -78,7 +78,7 @@ def make_layer(*, dataset, depth, season, variable, units) -> Layer:
             remotely sensed observations, also require a more recent
             climatology."""
         ),
-        tags=[],
+        packaging_tags=["core"],
         style=f"seawater_{variable}",
         input=LayerInput(
             dataset=dataset,

qgreenland/config/layers/Basemaps/land_ocean_shape.py

@@ -14,7 +14,7 @@ def make_land_ocean_layer(layer_id: str) -> Layer:
         id=layer_id,
         title=layer_id.capitalize(),
         description=(f"""Polygons representing the {layer_id}."""),
-        tags=[],
+        packaging_tags=["foo"] if layer_id == "land" else ["core", "foo"],
         style=layer_id,
         input=LayerInput(
             dataset=layer_params[layer_id],

qgreenland/config/layers/Reference/arctic_circle.py

@@ -11,7 +11,7 @@
         not set on the summer solstice, and does not rise on the winter
         solstice."""
     ),
-    tags=[],
+    packaging_tags=["core"],
     style="arctic_circle",
     input=LayerInput(
         dataset=dataset,

qgreenland/models/config/layer.py

@@ -36,11 +36,17 @@ class Layer(QgrBaseModel):
     description: str = Field(..., min_length=1)
     """Descriptive text shown as hover-text in the QGIS Layer Panel."""
 
+    # Temporary keeping these around for testing before config is fully migrated
+    in_package: bool = True
     tags: list[str] = []
-    """Additional categories that describe this data."""
 
-    in_package: bool = True
-    """Is this layer in the final QGreenland zip file?"""
+    packaging_tags: list[str] = ["core"]
+    """Which packages does this layer belong in? Layer will be omitted if empty.
+
+    Temporarily defaults to ["core"] to help us test more quickly.
+
+    TODO: Validate len > 0.
+    """
 
     show: bool = False
     """Is this layer initially "checked" or visible in QGIS?"""

qgreenland/test/constants.py

@@ -9,5 +9,6 @@
 TEST_CONFIG_W_MISSING_SETTINGS_DIR = TEST_DATA_DIR / "config_with_missing_settings"
 TEST_CONFIG_W_STRING_ORDER_VALUES = TEST_DATA_DIR / "config_with_string_order_values"
 
+# TODO: Handle new package-specific compile dirs
 MOCK_COMPILE_PACKAGE_DIR = TEST_DATA_DIR / "compile"
 MOCK_RELEASE_LAYERS_DIR = TEST_DATA_DIR / "release" / "layers"

qgreenland/test/util/test_misc.py

@@ -3,8 +3,8 @@
 
 
 def test_layer_compile_dir(raster_layer_node):
-    expected = COMPILE_PACKAGE_DIR / "Group" / "Subgroup" / "Example raster"
-    actual = layer_util.get_layer_compile_dir(raster_layer_node)
+    expected = COMPILE_PACKAGE_DIR / "foo" / "Group" / "Subgroup" / "Example raster"
+    actual = layer_util.get_layer_compile_dir(raster_layer_node, package_name="foo")
 
     assert expected == actual
 

qgreenland/util/config/export.py

@@ -22,61 +22,27 @@
     get_layer_release_filepath,
     vector_or_raster,
 )
-from qgreenland.util.metadata import build_layer_metadata
-from qgreenland.util.tree import LayerNode
-from qgreenland.util.version import get_build_version
-
-DEFAULT_LAYER_MANIFEST_PATH = Path("./layers.csv")
-
-
-def export_config_manifest(
-    cfg: Config,
-    output_path: Path = DEFAULT_LAYER_MANIFEST_PATH,
-) -> None:
-    """Write a machine-readable manifest to disk describing available layers.
-
-    This includes layers for which `in_package is False`.
-
-    This must be run after the layers are in their release location, because we
-    need to calculate their size on disk.
-    """
-    manifest_spec_version = "v0.1.0"
-    manifest = {
-        "version": manifest_spec_version,
-        "qgr_version": get_build_version(),
-        "layers": [
-            {
-                # ID first for readability
-                "id": layer_node.layer_cfg.id,
-                **layer_node.layer_cfg.dict(include={"title", "description", "tags"}),
-                "hierarchy": layer_node.group_name_path,
-                "layer_details": build_layer_metadata(layer_node.layer_cfg),
-                "assets": _layer_manifest_final_assets(layer_node),
-            }
-            for layer_node in cfg.layer_tree.leaves
-            # For now, do not include online layers in the layer manifest. The
-            # `QGreenland Custom` QGIS Plugin does not currently support online
-            # layers. Once online layers are supported in the plugin, this `if`
-            # statement can be removed.
-            if not isinstance(layer_node.layer_cfg.input.asset, OnlineAsset)
-        ],
-    }
-
-    with open(output_path, "w") as ofile:
-        json.dump(manifest, ofile)
+from qgreenland.util.tree import LayerNode, prune_layers_not_in_package
 
 
 def export_config_csv(
     cfg: Config,
-    output_path: Path = DEFAULT_LAYER_MANIFEST_PATH,
+    *,
+    output_path: Path,
+    package_name: str,
 ) -> None:
     """Write a report to disk summarizing layers in the zip package.
 
     This must be run after the layers are in their location, because we need to
     calculate their size on disk.
     """
+    layer_tree = prune_layers_not_in_package(
+        cfg.layer_tree,
+        package_name=package_name,
+    )
+
     report = []
-    for layer_node in cfg.layer_tree.leaves:
+    for layer_node in layer_tree.leaves:
         layer_cfg = layer_node.layer_cfg
 
         if not layer_cfg.in_package:
@@ -92,7 +58,10 @@ def export_config_csv(
             layer_size_bytes = 0
             internet_required = False
         else:
-            layer_fp = get_layer_compile_filepath(layer_node)
+            layer_fp = get_layer_compile_filepath(
+                layer_node,
+                package_name=package_name,
+            )
             layer_dir = layer_fp.parent
             layer_size_bytes = directory_size_bytes(layer_dir)
             internet_required = True

qgreenland/util/layer.py

@@ -3,29 +3,34 @@
 import qgreenland.exceptions as exc
 from qgreenland._typing import VectorOrRaster
 from qgreenland.constants.misc import PROVIDER_VECTOR_OR_RASTER_MAPPING
-from qgreenland.constants.paths import COMPILE_PACKAGE_DIR, RELEASE_LAYERS_DIR
+from qgreenland.constants.paths import RELEASE_LAYERS_DIR
 from qgreenland.models.config.asset import OnlineAsset
 from qgreenland.models.config.layer import Layer
 from qgreenland.util.fs import get_layer_fp
+from qgreenland.util.misc import compile_package_dir
 from qgreenland.util.tree import LayerNode
 
 
-def vector_or_raster(layer_node: LayerNode) -> VectorOrRaster:
+def vector_or_raster(
+    layer_node: LayerNode,
+) -> VectorOrRaster:
     layer_cfg = layer_node.layer_cfg
     if type(layer_cfg.input.asset) is OnlineAsset:
         return PROVIDER_VECTOR_OR_RASTER_MAPPING[layer_cfg.input.asset.provider]
     else:
-        layer_path = get_layer_compile_filepath(layer_node)
+        layer_path = get_layer_release_filepath(layer_node)
         return _vector_or_raster_from_fp(layer_path)
 
 
 def get_layer_compile_dir(
     layer_node: LayerNode,
+    *,
+    package_name: str,
 ) -> Path:
     """Get the layer directory in package compilation location."""
     layer_group_path_str = "/".join(layer_node.group_name_path)
     return (
-        COMPILE_PACKAGE_DIR
+        compile_package_dir(package_name)
         / layer_group_path_str
         / _layer_dirname_from_cfg(layer_node.layer_cfg)
     )
@@ -41,10 +46,13 @@ def datasource_dirname(*, dataset_id: str, asset_id: str) -> str:
     return f"{dataset_id}.{asset_id}"
 
 
-def get_layer_compile_filepath(
-    layer_node: LayerNode,
-) -> Path:
-    return get_layer_fp(get_layer_compile_dir(layer_node))
+def get_layer_compile_filepath(layer_node: LayerNode, *, package_name: str) -> Path:
+    return get_layer_fp(
+        get_layer_compile_dir(
+            layer_node,
+            package_name=package_name,
+        )
+    )
 
 
 def get_layer_release_filepath(

qgreenland/util/luigi/__init__.py

@@ -69,6 +69,7 @@ def fetch_tasks_from_dataset(
 @cache
 def generate_layer_pipelines(
     *,
+    package_name: str,
     fetch_only: bool = False,
 ) -> list[luigi.Task]:
     """Generate a list of pre-configured tasks based on layer configuration.
@@ -87,6 +88,10 @@ def generate_layer_pipelines(
         if isinstance(layer_cfg.input.asset, OnlineAsset):
             continue
 
+        # Check if the layer is meant to be in `package_name`.
+        if package_name not in layer_cfg.packaging_tags:
+            continue
+
         # Create tasks, making each task dependent on the previous task.
         task = fetch_task_from_layer(layer_cfg)
         if fetch_only: