From 0a8ad344ac079bdbd8e17d51d4b7ff5653910a70 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Tue, 15 Aug 2023 16:31:21 -0600 Subject: [PATCH 01/11] Cleanup OBE references to `gdal_remote` and "Online" QgsLayerType --- qgreenland/_typing.py | 2 +- qgreenland/models/config/asset.py | 4 +--- qgreenland/test/util/test_misc.py | 2 +- qgreenland/util/config/export.py | 6 ++++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/qgreenland/_typing.py b/qgreenland/_typing.py index 0d0683fd..c4bc6903 100644 --- a/qgreenland/_typing.py +++ b/qgreenland/_typing.py @@ -5,7 +5,7 @@ """ from typing import Literal, Union -QgsLayerType = Literal["Vector", "Raster", "Online"] +QgsLayerType = Literal["Vector", "Raster"] QgsLayerProviderType = Literal["gdal", "ogr", "wms", "wfs", "wcs"] ResamplingMethod = Literal["bilinear", "nearest"] diff --git a/qgreenland/models/config/asset.py b/qgreenland/models/config/asset.py index d0f1d90d..fb3490d0 100644 --- a/qgreenland/models/config/asset.py +++ b/qgreenland/models/config/asset.py @@ -44,9 +44,7 @@ def provenance(self) -> str: return f"# Data fetched via HTTP from {[str(u) for u in self.urls]}" -# TODO: OnlineRaster/OnlineVector asset types? The thing that makes this a -# "gdal_remote" layer is the `/vsicurl/` prefix. Otherwise, this is created as a -# regular layer with a URL as its path. +# TODO: OnlineRaster/OnlineVector asset types? class OnlineAsset(DatasetAsset): """A QGIS online layer that is not fetched, but is accessed by QGIS.""" diff --git a/qgreenland/test/util/test_misc.py b/qgreenland/test/util/test_misc.py index 53f11188..ab6e6d78 100644 --- a/qgreenland/test/util/test_misc.py +++ b/qgreenland/test/util/test_misc.py @@ -9,5 +9,5 @@ def test_layer_compile_dir(raster_layer_node): assert expected == actual -def test_vector_or_raster_gdal_remote(online_layer_node): +def test_vector_or_raster(online_layer_node): assert layer_util.vector_or_raster(online_layer_node) == "Raster" diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index cd1f4fd6..b19b5cf2 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -8,7 +8,7 @@ import json import os from pathlib import Path -from typing import Union +from typing import Literal, Union from humanize import naturalsize @@ -82,7 +82,9 @@ def export_config_csv( if not layer_cfg.in_package: continue - layer_type: QgsLayerType + # NOTE: we want to make it clear to users that some layers are "online" + # and we use the "Vector or Raster" column to do that. + layer_type: Union[QgsLayerType, Literal["Online"]] if isinstance(layer_cfg.input.asset, OnlineAsset): layer_type = "Online" # Online layers have no size on disk. From 3e073a3e94c4c6b52da4ba330f7ecd6d7d2e3e56 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Tue, 15 Aug 2023 16:37:25 -0600 Subject: [PATCH 02/11] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f2295e..8fdf84dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ places" layer. Scale point markers for this layer by population. * Add new "Internet-required data/Geological Map (1:500 000)" layer from GEUS. * Add new "Geology/Mineral occurrences" layer from GEUS. +* Remove OBE references to "gdal_remote" layers. Remove "Online" from `QgsLayerType`. # v3.0.0alpha4 (2023-07-21) From 78d503a05744d112c99a044c1f4e9041c7bff6aa Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:30:38 -0600 Subject: [PATCH 03/11] Remove dev-specific content from CHANGELOG --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fdf84dd..95f2295e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,6 @@ places" layer. Scale point markers for this layer by population. * Add new "Internet-required data/Geological Map (1:500 000)" layer from GEUS. * Add new "Geology/Mineral occurrences" layer from GEUS. -* Remove OBE references to "gdal_remote" layers. Remove "Online" from `QgsLayerType`. # v3.0.0alpha4 (2023-07-21) From 4ac5f03517c442f09a3b1c677fcd1e82505444f5 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:31:16 -0600 Subject: [PATCH 04/11] Clarify when CHANGLEOG should be updated in pr template --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a9cd8bb8..fc6cb151 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,6 +13,6 @@ checkbox. - [ ] Config lockfile updated (`inv config.export > qgreenland/config/cfg-lock.json`) - [ ] Environment lockfile updated if needed (`conda-lock`) - [ ] Version bumped if needed (`bumpversion (major|minor|patch|prerelease|build)`) -- [ ] CHANGELOG.md updated +- [ ] CHANGELOG.md updated (for user-facing changes) - [ ] Documentation updated if needed - [ ] New unit tests if needed From f4597b0ff06d4860b17f442bebb511a336b34547 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:36:39 -0600 Subject: [PATCH 05/11] LayerType -> VectorOrRaster --- qgreenland/_typing.py | 2 +- qgreenland/constants/misc.py | 4 ++-- qgreenland/util/config/export.py | 4 ++-- qgreenland/util/layer.py | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/qgreenland/_typing.py b/qgreenland/_typing.py index c4bc6903..f26810d5 100644 --- a/qgreenland/_typing.py +++ b/qgreenland/_typing.py @@ -5,7 +5,7 @@ """ from typing import Literal, Union -QgsLayerType = Literal["Vector", "Raster"] +VectorOrRaster = Literal["Vector", "Raster"] QgsLayerProviderType = Literal["gdal", "ogr", "wms", "wfs", "wcs"] ResamplingMethod = Literal["bilinear", "nearest"] diff --git a/qgreenland/constants/misc.py b/qgreenland/constants/misc.py index cf4635c4..a27e907e 100644 --- a/qgreenland/constants/misc.py +++ b/qgreenland/constants/misc.py @@ -1,6 +1,6 @@ -from qgreenland._typing import QgsLayerProviderType, QgsLayerType +from qgreenland._typing import QgsLayerProviderType, VectorOrRaster -PROVIDER_LAYERTYPE_MAPPING: dict[QgsLayerProviderType, QgsLayerType] = { +PROVIDER_VECTOR_OR_RASTER_MAPPING: dict[QgsLayerProviderType, VectorOrRaster] = { "gdal": "Raster", "wms": "Raster", "wfs": "Vector", diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index b19b5cf2..11e0b7f2 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -12,7 +12,7 @@ from humanize import naturalsize -from qgreenland._typing import QgsLayerType +from qgreenland._typing import VectorOrRaster from qgreenland.models.config import Config from qgreenland.models.config.asset import OnlineAsset from qgreenland.util.fs import directory_contents, directory_size_bytes @@ -84,7 +84,7 @@ def export_config_csv( # NOTE: we want to make it clear to users that some layers are "online" # and we use the "Vector or Raster" column to do that. - layer_type: Union[QgsLayerType, Literal["Online"]] + layer_type: Union[VectorOrRaster, Literal["Online"]] if isinstance(layer_cfg.input.asset, OnlineAsset): layer_type = "Online" # Online layers have no size on disk. diff --git a/qgreenland/util/layer.py b/qgreenland/util/layer.py index 2e309efd..d7677b71 100644 --- a/qgreenland/util/layer.py +++ b/qgreenland/util/layer.py @@ -1,8 +1,8 @@ from pathlib import Path import qgreenland.exceptions as exc -from qgreenland._typing import QgsLayerType -from qgreenland.constants.misc import PROVIDER_LAYERTYPE_MAPPING +from qgreenland._typing import VectorOrRaster +from qgreenland.constants.misc import PROVIDER_VECTOR_OR_RASTER_MAPPING from qgreenland.constants.paths import COMPILE_PACKAGE_DIR, RELEASE_LAYERS_DIR from qgreenland.models.config.asset import OnlineAsset from qgreenland.models.config.layer import Layer @@ -10,10 +10,10 @@ from qgreenland.util.tree import LayerNode -def vector_or_raster(layer_node: LayerNode) -> QgsLayerType: +def vector_or_raster(layer_node: LayerNode) -> VectorOrRaster: layer_cfg = layer_node.layer_cfg if type(layer_cfg.input.asset) is OnlineAsset: - return PROVIDER_LAYERTYPE_MAPPING[layer_cfg.input.asset.provider] + return PROVIDER_VECTOR_OR_RASTER_MAPPING[layer_cfg.input.asset.provider] else: layer_path = get_layer_compile_filepath(layer_node) return _vector_or_raster_from_fp(layer_path) @@ -57,7 +57,7 @@ def _layer_dirname_from_cfg(layer_cfg: Layer) -> str: return layer_cfg.title -def _vector_or_raster_from_fp(fp: Path) -> QgsLayerType: +def _vector_or_raster_from_fp(fp: Path) -> VectorOrRaster: if fp.suffix == ".tif": return "Raster" elif fp.suffix == ".gpkg": From 74f4094ca65b4a2d7bf11acacf2f097eb219e810 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:42:15 -0600 Subject: [PATCH 06/11] layers_list.csv: separate column for identifying online-only layers --- qgreenland/util/config/export.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index 11e0b7f2..e33ae90d 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -8,7 +8,7 @@ import json import os from pathlib import Path -from typing import Literal, Union +from typing import Union from humanize import naturalsize @@ -82,18 +82,20 @@ def export_config_csv( if not layer_cfg.in_package: continue - # NOTE: we want to make it clear to users that some layers are "online" - # and we use the "Vector or Raster" column to do that. - layer_type: Union[VectorOrRaster, Literal["Online"]] + vector_or_raster_data: VectorOrRaster + layer_data_on_disk: bool + if isinstance(layer_cfg.input.asset, OnlineAsset): - layer_type = "Online" + vector_or_raster_data = "Online" # Online layers have no size on disk. layer_size_bytes = 0 + layer_data_on_disk = False else: layer_fp = get_layer_compile_filepath(layer_node) layer_dir = layer_fp.parent layer_size_bytes = directory_size_bytes(layer_dir) - layer_type = vector_or_raster(layer_node) + vector_or_raster_data = vector_or_raster(layer_node) + layer_data_on_disk = True dataset_cfg = layer_cfg.input.dataset @@ -103,13 +105,14 @@ def export_config_csv( "Subgroup": "/".join(layer_node.group_name_path[1:]), "Layer Title": layer_cfg.title, "Layer Description": layer_cfg.description, - "Vector or Raster": layer_type, + "Vector or Raster": vector_or_raster_data, "Data Source Title": dataset_cfg.metadata.title, "Data Source Abstract": dataset_cfg.metadata.abstract, "Data Source Citation": dataset_cfg.metadata.citation.text, "Data Source Citation URL": dataset_cfg.metadata.citation.url, "Layer Size": naturalsize(layer_size_bytes), "Layer Size Bytes": layer_size_bytes, + "Layer Data On Disk": layer_data_on_disk, } ) From ca8cf285e7b9cd20dde3fb47171e7451ef492a0c Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:51:44 -0600 Subject: [PATCH 07/11] layer_csv.csv: identify if online layers are raster or vector --- qgreenland/util/config/export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index e33ae90d..be76cfac 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -85,8 +85,9 @@ def export_config_csv( vector_or_raster_data: VectorOrRaster layer_data_on_disk: bool + vector_or_raster_data = vector_or_raster(layer_node) + if isinstance(layer_cfg.input.asset, OnlineAsset): - vector_or_raster_data = "Online" # Online layers have no size on disk. layer_size_bytes = 0 layer_data_on_disk = False @@ -94,7 +95,6 @@ def export_config_csv( layer_fp = get_layer_compile_filepath(layer_node) layer_dir = layer_fp.parent layer_size_bytes = directory_size_bytes(layer_dir) - vector_or_raster_data = vector_or_raster(layer_node) layer_data_on_disk = True dataset_cfg = layer_cfg.input.dataset From b1eb506b9cfab131d2aa4c6c1d19c8941d899e91 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 16:52:19 -0600 Subject: [PATCH 08/11] CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f2295e..0253fe6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ places" layer. Scale point markers for this layer by population. * Add new "Internet-required data/Geological Map (1:500 000)" layer from GEUS. * Add new "Geology/Mineral occurrences" layer from GEUS. +* Update `layer_list.csv` to include new column indicating if each layer is + stored on disk. Internet-required layers take the value `False`. # v3.0.0alpha4 (2023-07-21) From 907676b1ecedc02603ea8d5c870dc95278c924fb Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Wed, 16 Aug 2023 17:02:01 -0600 Subject: [PATCH 09/11] Fixup config export test --- qgreenland/test/util/config/test_config_export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qgreenland/test/util/config/test_config_export.py b/qgreenland/test/util/config/test_config_export.py index c2a7b845..dcb7ba21 100644 --- a/qgreenland/test/util/config/test_config_export.py +++ b/qgreenland/test/util/config/test_config_export.py @@ -117,7 +117,8 @@ def test_export_config_csv(full_cfg): { **common, "Layer Title": "Example online", - "Vector or Raster": "Online", + "Vector or Raster": "Raster", + "Layer Data On Disk": "False", }, { **common, @@ -125,6 +126,7 @@ def test_export_config_csv(full_cfg): "Vector or Raster": "Raster", "Layer Size": "619 Bytes", "Layer Size Bytes": "619", + "Layer Data On Disk": "True", }, ] From b9f10fa24e56194daa74065ed32dad5adaa2b29c Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Thu, 17 Aug 2023 10:33:48 -0600 Subject: [PATCH 10/11] layer-list.csv: `Layer Data On Disk` -> `Internet Required?` --- qgreenland/util/config/export.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index be76cfac..6912fcf1 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -83,19 +83,19 @@ def export_config_csv( continue vector_or_raster_data: VectorOrRaster - layer_data_on_disk: bool + internet_required: bool vector_or_raster_data = vector_or_raster(layer_node) if isinstance(layer_cfg.input.asset, OnlineAsset): # Online layers have no size on disk. layer_size_bytes = 0 - layer_data_on_disk = False + internet_required = False else: layer_fp = get_layer_compile_filepath(layer_node) layer_dir = layer_fp.parent layer_size_bytes = directory_size_bytes(layer_dir) - layer_data_on_disk = True + internet_required = True dataset_cfg = layer_cfg.input.dataset @@ -112,7 +112,7 @@ def export_config_csv( "Data Source Citation URL": dataset_cfg.metadata.citation.url, "Layer Size": naturalsize(layer_size_bytes), "Layer Size Bytes": layer_size_bytes, - "Layer Data On Disk": layer_data_on_disk, + "Internet Required?": internet_required, } ) From 46b14c9f036a28be03800f85144f8e01756b37d0 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Thu, 17 Aug 2023 10:34:53 -0600 Subject: [PATCH 11/11] Fixup config export test --- qgreenland/test/util/config/test_config_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qgreenland/test/util/config/test_config_export.py b/qgreenland/test/util/config/test_config_export.py index dcb7ba21..a9b5f6f4 100644 --- a/qgreenland/test/util/config/test_config_export.py +++ b/qgreenland/test/util/config/test_config_export.py @@ -118,7 +118,7 @@ def test_export_config_csv(full_cfg): **common, "Layer Title": "Example online", "Vector or Raster": "Raster", - "Layer Data On Disk": "False", + "Internet Required?": "False", }, { **common, @@ -126,7 +126,7 @@ def test_export_config_csv(full_cfg): "Vector or Raster": "Raster", "Layer Size": "619 Bytes", "Layer Size Bytes": "619", - "Layer Data On Disk": "True", + "Internet Required?": "True", }, ]