From bae913f624bdb387718d95616bb9efab8c31af99 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 29 Mar 2024 10:59:38 -0400 Subject: [PATCH 01/17] ENH: clarify one/zero-based numbering --- event_model/schemas/stream_datum.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/event_model/schemas/stream_datum.json b/event_model/schemas/stream_datum.json index d5217f39..cd9fa6a8 100644 --- a/event_model/schemas/stream_datum.json +++ b/event_model/schemas/stream_datum.json @@ -32,7 +32,7 @@ "type": "string" }, "indices": { - "description": "A slice object passed to the StreamResource handler so it can hand back data and timestamps", + "description": "A slice object passed to the StreamResource handler so it can hand back data and timestamps (zero-based)", "allOf": [ { "$ref": "#/$defs/StreamRange" @@ -40,7 +40,7 @@ ] }, "seq_nums": { - "description": "A slice object showing the Event numbers the resource corresponds to", + "description": "A slice object showing the Event numbers the resource corresponds to (one-based)", "allOf": [ { "$ref": "#/$defs/StreamRange" From c9a1f1255dbce9de33f47389b664ca452a7c6573 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 29 Mar 2024 11:30:18 -0400 Subject: [PATCH 02/17] MNT: remove the field --- docs/user/explanations/data-model.rst | 5 +---- docs/user/explanations/external.rst | 1 - event_model/__init__.py | 15 --------------- event_model/documents/resource.py | 6 ------ event_model/documents/stream_resource.py | 6 ------ event_model/schemas/resource.json | 9 --------- event_model/schemas/stream_datum.json | 4 ++-- event_model/schemas/stream_resource.json | 9 --------- 8 files changed, 3 insertions(+), 52 deletions(-) diff --git a/docs/user/explanations/data-model.rst b/docs/user/explanations/data-model.rst index 3687d688..3a637325 100644 --- a/docs/user/explanations/data-model.rst +++ b/docs/user/explanations/data-model.rst @@ -390,8 +390,7 @@ Minimal nontrivial valid example: .. code-block:: python # 'resource' document - {'path_semantics': 'posix', - 'resource_kwargs': {}, + {'resource_kwargs': {}, 'resource_path': '/local/path/subdirectory/data_file', 'root': '/local/path/', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd', @@ -407,7 +406,6 @@ Typical example: 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', 'resource_kwargs': {'frame_per_point': 10}, - 'path_semantics': 'posix', 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} @@ -487,7 +485,6 @@ Typical example: 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', 'resource_kwargs': {'frame_per_point': 1}, - 'path_semantics': 'posix', 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} diff --git a/docs/user/explanations/external.rst b/docs/user/explanations/external.rst index 757262dd..f339cd07 100644 --- a/docs/user/explanations/external.rst +++ b/docs/user/explanations/external.rst @@ -96,7 +96,6 @@ contains path-related details. 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', 'resource_kwargs': {'frame_per_point': 10}, - 'path_semantics': 'posix', 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} diff --git a/event_model/__init__.py b/event_model/__init__.py index b7ca1608..f0014771 100644 --- a/event_model/__init__.py +++ b/event_model/__init__.py @@ -1899,13 +1899,6 @@ def __iter__(self) -> Iterator: ) -PATH_SEMANTICS: Dict[str, Literal["posix", "windows"]] = { - "posix": "posix", - "nt": "windows", -} -default_path_semantics: Literal["posix", "windows"] = PATH_SEMANTICS[os.name] - - @dataclass class ComposeResource: start: Optional[RunStart] @@ -1916,7 +1909,6 @@ def __call__( root: str, resource_path: str, resource_kwargs: Dict[str, Any], - path_semantics: Literal["posix", "windows"] = default_path_semantics, uid: Optional[str] = None, validate: bool = True, ) -> ComposeResourceBundle: @@ -1924,7 +1916,6 @@ def __call__( uid = str(uuid.uuid4()) doc = Resource( - path_semantics=path_semantics, uid=uid, spec=spec, root=root, @@ -1952,7 +1943,6 @@ def compose_resource( root: str, resource_path: str, resource_kwargs: Dict[str, Any], - path_semantics: Literal["posix", "windows"] = default_path_semantics, start: Optional[RunStart] = None, uid: Optional[str] = None, validate: bool = True, @@ -1965,7 +1955,6 @@ def compose_resource( root, resource_path, resource_kwargs, - path_semantics=path_semantics, uid=uid, validate=validate, ) @@ -2052,7 +2041,6 @@ def __call__( resource_path: str, data_key: str, resource_kwargs: Dict[str, Any], - path_semantics: Literal["posix", "windows"] = default_path_semantics, uid: Optional[str] = None, validate: bool = True, ) -> ComposeStreamResourceBundle: @@ -2066,7 +2054,6 @@ def __call__( root=root, resource_path=resource_path, resource_kwargs=resource_kwargs, - path_semantics=path_semantics, ) if self.start: @@ -2091,7 +2078,6 @@ def compose_stream_resource( resource_path: str, data_key: str, resource_kwargs: Dict[str, Any], - path_semantics: Literal["posix", "windows"] = default_path_semantics, start: Optional[RunStart] = None, uid: Optional[str] = None, validate: bool = True, @@ -2105,7 +2091,6 @@ def compose_stream_resource( resource_path, data_key, resource_kwargs, - path_semantics=path_semantics, uid=uid, validate=validate, ) diff --git a/event_model/documents/resource.py b/event_model/documents/resource.py index 882ceb69..b8f4f949 100644 --- a/event_model/documents/resource.py +++ b/event_model/documents/resource.py @@ -43,12 +43,6 @@ class Resource(PartialResource): externally-stored data """ - path_semantics: NotRequired[ - Annotated[ - Literal["posix", "windows"], - Field(description="Rules for joining paths"), - ] - ] run_start: NotRequired[ Annotated[ str, diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index 3ab1877e..a1e53a24 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -14,12 +14,6 @@ class StreamResource(TypedDict): externally-stored data streams """ - path_semantics: NotRequired[ - Annotated[ - Literal["posix", "windows"], - Field(description="Rules for joining paths"), - ] - ] data_key: Annotated[ str, Field( diff --git a/event_model/schemas/resource.json b/event_model/schemas/resource.json index ccbb4924..a143be46 100644 --- a/event_model/schemas/resource.json +++ b/event_model/schemas/resource.json @@ -3,15 +3,6 @@ "description": "Document to reference a collection (e.g. file or group of files) of externally-stored data", "type": "object", "properties": { - "path_semantics": { - "title": "Path Semantics", - "description": "Rules for joining paths", - "type": "string", - "enum": [ - "posix", - "windows" - ] - }, "resource_kwargs": { "title": "Resource Kwargs", "description": "Additional argument to pass to the Handler to read a Resource", diff --git a/event_model/schemas/stream_datum.json b/event_model/schemas/stream_datum.json index cd9fa6a8..d5217f39 100644 --- a/event_model/schemas/stream_datum.json +++ b/event_model/schemas/stream_datum.json @@ -32,7 +32,7 @@ "type": "string" }, "indices": { - "description": "A slice object passed to the StreamResource handler so it can hand back data and timestamps (zero-based)", + "description": "A slice object passed to the StreamResource handler so it can hand back data and timestamps", "allOf": [ { "$ref": "#/$defs/StreamRange" @@ -40,7 +40,7 @@ ] }, "seq_nums": { - "description": "A slice object showing the Event numbers the resource corresponds to (one-based)", + "description": "A slice object showing the Event numbers the resource corresponds to", "allOf": [ { "$ref": "#/$defs/StreamRange" diff --git a/event_model/schemas/stream_resource.json b/event_model/schemas/stream_resource.json index f5a9a02f..b2076322 100644 --- a/event_model/schemas/stream_resource.json +++ b/event_model/schemas/stream_resource.json @@ -8,15 +8,6 @@ "description": "A string to show which data_key of the Descriptor are being streamed", "type": "string" }, - "path_semantics": { - "title": "Path Semantics", - "description": "Rules for joining paths", - "type": "string", - "enum": [ - "posix", - "windows" - ] - }, "resource_kwargs": { "title": "Resource Kwargs", "description": "Additional argument to pass to the Handler to read a Stream Resource", From 56ebabe747194e005e3e528cb8fcf5e3efdc38b1 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 29 Mar 2024 11:48:59 -0400 Subject: [PATCH 03/17] STY: fix linting --- event_model/documents/resource.py | 2 +- event_model/documents/stream_resource.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/event_model/documents/resource.py b/event_model/documents/resource.py index b8f4f949..a9a1ba07 100644 --- a/event_model/documents/resource.py +++ b/event_model/documents/resource.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from typing_extensions import Annotated, Literal, NotRequired, TypedDict +from typing_extensions import Annotated, NotRequired, TypedDict from .generate.type_wrapper import Field, add_extra_schema diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index a1e53a24..16a11463 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from typing_extensions import Annotated, Literal, NotRequired, TypedDict +from typing_extensions import Annotated, NotRequired, TypedDict from .generate.type_wrapper import Field, add_extra_schema From 43e135c30b4e2709e62f6992390b80132870453b Mon Sep 17 00:00:00 2001 From: Eugene M Date: Fri, 29 Mar 2024 11:57:54 -0400 Subject: [PATCH 04/17] MNT: renamed to --- docs/user/explanations/data-model.rst | 6 +++--- docs/user/explanations/external.rst | 12 ++++++------ event_model/__init__.py | 22 +++++++++++----------- event_model/documents/resource.py | 4 ++-- event_model/documents/stream_resource.py | 4 ++-- event_model/schemas/resource.json | 8 ++++---- event_model/schemas/stream_resource.json | 8 ++++---- event_model/tests/test_em.py | 20 ++++++++++---------- event_model/tests/test_filler.py | 2 +- event_model/tests/test_run_router.py | 4 ++-- 10 files changed, 45 insertions(+), 45 deletions(-) diff --git a/docs/user/explanations/data-model.rst b/docs/user/explanations/data-model.rst index 3a637325..21fb3ec6 100644 --- a/docs/user/explanations/data-model.rst +++ b/docs/user/explanations/data-model.rst @@ -390,7 +390,7 @@ Minimal nontrivial valid example: .. code-block:: python # 'resource' document - {'resource_kwargs': {}, + {'parameters': {}, 'resource_path': '/local/path/subdirectory/data_file', 'root': '/local/path/', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd', @@ -405,7 +405,7 @@ Typical example: {'spec': 'AD_HDF5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', - 'resource_kwargs': {'frame_per_point': 10}, + 'parameters': {'frame_per_point': 10}, 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} @@ -484,7 +484,7 @@ Typical example: 'spec': 'AD_HDF5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', - 'resource_kwargs': {'frame_per_point': 1}, + 'parameters': {'frame_per_point': 1}, 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} diff --git a/docs/user/explanations/external.rst b/docs/user/explanations/external.rst index f339cd07..0c949afe 100644 --- a/docs/user/explanations/external.rst +++ b/docs/user/explanations/external.rst @@ -95,7 +95,7 @@ contains path-related details. 'spec': 'AD_HDF5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', - 'resource_kwargs': {'frame_per_point': 10}, + 'parameters': {'frame_per_point': 10}, 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} @@ -105,7 +105,7 @@ context-dependent (depending on what system you are accessing the data from) and subject to change (if the data is moved over time). The ``spec`` gives us a hint about the format of this asset, whether it be a -file, multiple files, or something more specialized. The ``resource_kwargs`` +file, multiple files, or something more specialized. The ``parameters`` provide any additional parameters for reading it. Handlers @@ -120,7 +120,7 @@ Handler Interface A 'handler class' may be any callable with the signature:: - handler_class(full_path, **resource_kwargs) + handler_class(full_path, **parameters) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -134,8 +134,8 @@ typically implemented using a class that implements ``__init__`` and .. code:: python class MyHandler: - def __init__(self, path, **resource_kwargs): - # Consume the path information and the 'resource_kwargs' from the + def __init__(self, path, **parameters): + # Consume the path information and the 'parameters' kwargs from the # Resource. Typically stashes some state and/or opens file(s). ... @@ -150,7 +150,7 @@ But in general it may be any callable-that-returns-a-callable. .. code:: python - def handler(path, **resource_kwargs): + def handler(path, **parameters): def f(**datum_kwargs): return some_array_like return f diff --git a/event_model/__init__.py b/event_model/__init__.py index f0014771..a71dbce2 100644 --- a/event_model/__init__.py +++ b/event_model/__init__.py @@ -538,7 +538,7 @@ class Filler(DocumentRouter): A 'handler class' may be any callable with the signature:: - handler_class(full_path, **resource_kwargs) + handler_class(full_path, **parameters) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -1041,7 +1041,7 @@ def get_handler(self, resource: Resource) -> Any: handler = _attempt_with_retries( func=handler_class, args=(resource_path,), - kwargs=resource["resource_kwargs"], + kwargs=resource["parameters"], intervals=[0] + self.retry_intervals, error_to_catch=IOError, error_to_raise=error_to_raise, @@ -1422,7 +1422,7 @@ class RunRouter(DocumentRouter): A 'handler class' may be any callable with the signature:: - handler_class(full_path, **resource_kwargs) + handler_class(full_path, **parameters) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -1908,7 +1908,7 @@ def __call__( spec: str, root: str, resource_path: str, - resource_kwargs: Dict[str, Any], + parameters: Dict[str, Any], uid: Optional[str] = None, validate: bool = True, ) -> ComposeResourceBundle: @@ -1919,7 +1919,7 @@ def __call__( uid=uid, spec=spec, root=root, - resource_kwargs=resource_kwargs, + parameters=parameters, resource_path=resource_path, ) @@ -1942,7 +1942,7 @@ def compose_resource( spec: str, root: str, resource_path: str, - resource_kwargs: Dict[str, Any], + parameters: Dict[str, Any], start: Optional[RunStart] = None, uid: Optional[str] = None, validate: bool = True, @@ -1954,7 +1954,7 @@ def compose_resource( spec, root, resource_path, - resource_kwargs, + parameters, uid=uid, validate=validate, ) @@ -2040,7 +2040,7 @@ def __call__( root: str, resource_path: str, data_key: str, - resource_kwargs: Dict[str, Any], + parameters: Dict[str, Any], uid: Optional[str] = None, validate: bool = True, ) -> ComposeStreamResourceBundle: @@ -2053,7 +2053,7 @@ def __call__( spec=spec, root=root, resource_path=resource_path, - resource_kwargs=resource_kwargs, + parameters=parameters, ) if self.start: @@ -2077,7 +2077,7 @@ def compose_stream_resource( root: str, resource_path: str, data_key: str, - resource_kwargs: Dict[str, Any], + parameters: Dict[str, Any], start: Optional[RunStart] = None, uid: Optional[str] = None, validate: bool = True, @@ -2090,7 +2090,7 @@ def compose_stream_resource( root, resource_path, data_key, - resource_kwargs, + parameters, uid=uid, validate=validate, ) diff --git a/event_model/documents/resource.py b/event_model/documents/resource.py index a9a1ba07..fb9227ba 100644 --- a/event_model/documents/resource.py +++ b/event_model/documents/resource.py @@ -18,10 +18,10 @@ class PartialResource(TypedDict): resource_path: Annotated[ str, Field(description="Filepath or URI for locating this resource") ] - resource_kwargs: Annotated[ + parameters: Annotated[ Dict[str, Any], Field( - description="Additional argument to pass to the Handler to read a Resource" + description="Additional keyword arguments to pass to the Handler to read a Resource" ), ] diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index 16a11463..4dd1f03b 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -21,10 +21,10 @@ class StreamResource(TypedDict): "Descriptor are being streamed" ), ] - resource_kwargs: Annotated[ + parameters: Annotated[ Dict[str, Any], Field( - description="Additional argument to pass to the Handler to read a " + description="Additional keyword arguments to pass to the Handler to read a " "Stream Resource", ), ] diff --git a/event_model/schemas/resource.json b/event_model/schemas/resource.json index a143be46..7df1b449 100644 --- a/event_model/schemas/resource.json +++ b/event_model/schemas/resource.json @@ -3,9 +3,9 @@ "description": "Document to reference a collection (e.g. file or group of files) of externally-stored data", "type": "object", "properties": { - "resource_kwargs": { - "title": "Resource Kwargs", - "description": "Additional argument to pass to the Handler to read a Resource", + "parameters": { + "title": "Parameters", + "description": "Additional keyword arguments to pass to the Handler to read a Resource", "type": "object" }, "resource_path": { @@ -35,7 +35,7 @@ } }, "required": [ - "resource_kwargs", + "parameters", "resource_path", "root", "spec", diff --git a/event_model/schemas/stream_resource.json b/event_model/schemas/stream_resource.json index b2076322..43e8ac23 100644 --- a/event_model/schemas/stream_resource.json +++ b/event_model/schemas/stream_resource.json @@ -8,9 +8,9 @@ "description": "A string to show which data_key of the Descriptor are being streamed", "type": "string" }, - "resource_kwargs": { - "title": "Resource Kwargs", - "description": "Additional argument to pass to the Handler to read a Stream Resource", + "parameters": { + "title": "Parameters", + "description": "Additional keyword arguments to pass to the Handler to read a Stream Resource", "type": "object" }, "resource_path": { @@ -41,7 +41,7 @@ }, "required": [ "data_key", - "resource_kwargs", + "parameters", "resource_path", "root", "spec", diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index a18caa5c..5c6babce 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -71,7 +71,7 @@ def test_compose_run(): assert bundle.compose_event is compose_event assert bundle.compose_event_page is compose_event_page bundle = compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) resource_doc, compose_datum, compose_datum_page = bundle assert bundle.resource_doc is resource_doc @@ -111,7 +111,7 @@ def test_compose_stream_resource(tmp_path): root=str(tmp_path), data_key="det1", resource_path="test_streams", - resource_kwargs={}, + parameters={}, ) resource_doc, compose_stream_datum = bundle assert bundle.stream_resource_doc is resource_doc @@ -134,7 +134,7 @@ def test_round_trip_pagination(): name="primary", ) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -244,7 +244,7 @@ def test_bulk_events_to_event_page(tmp_path): path_root = str(tmp_path) res_bundle = run_bundle.compose_resource( - spec="TIFF", root=path_root, resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root=path_root, resource_path="stack.tiff", parameters={} ) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -315,7 +315,7 @@ def test_sanitize_doc(): def test_bulk_datum_to_datum_page(): run_bundle = event_model.compose_run() res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) datum1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -353,7 +353,7 @@ def test_document_router_smoke_test(): ) dr("descriptor", desc_bundle_baseline.descriptor_doc) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) dr("resource", res_bundle.resource_doc) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -392,7 +392,7 @@ def test_document_router_streams_smoke_test(tmp_path): data_key="det1", root=str(tmp_path), resource_path="test_streams", - resource_kwargs={}, + parameters={}, ) dr("stream_resource", stream_resource_doc) datum_doc = compose_stream_datum( @@ -425,7 +425,7 @@ def test_document_router_with_validation(): ) dr("descriptor", desc_bundle_baseline.descriptor_doc, validate=True) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) dr("resource", res_bundle.resource_doc, validate=True) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -762,7 +762,7 @@ def test_single_run_document_router(): ) sr("descriptor", desc_bundle_baseline.descriptor_doc) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} ) sr("resource", res_bundle.resource_doc) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -1021,5 +1021,5 @@ def test_array_like(): def test_resource_start_optional(): event_model.compose_resource( - spec="TEST", root="/", resource_path="", resource_kwargs={} + spec="TEST", root="/", resource_path="", parameters={} ) diff --git a/event_model/tests/test_filler.py b/event_model/tests/test_filler.py index 369fdabd..c1d6c660 100644 --- a/event_model/tests/test_filler.py +++ b/event_model/tests/test_filler.py @@ -28,7 +28,7 @@ spec="DUMMY", root=str(path_root), resource_path="stack.tiff", - resource_kwargs={"a": 1, "b": 2}, + parameters={"a": 1, "b": 2}, ) datum_doc = res_bundle.compose_datum(datum_kwargs={"c": 3, "d": 4}) raw_event = desc_bundle.compose_event( diff --git a/event_model/tests/test_run_router.py b/event_model/tests/test_run_router.py index 62405496..ef78ca06 100644 --- a/event_model/tests/test_run_router.py +++ b/event_model/tests/test_run_router.py @@ -33,7 +33,7 @@ def test_run_router(tmp_path): baseline_descriptor_doc, compose_baseline_event, compose_event_page = bundle docs.append(("descriptor", baseline_descriptor_doc)) bundle = compose_resource( - spec="TIFF", root=str(tmp_path), resource_path="stack.tiff", resource_kwargs={} + spec="TIFF", root=str(tmp_path), resource_path="stack.tiff", parameters={} ) resource_doc, compose_datum, compose_datum_page = bundle docs.append(("resource", resource_doc)) @@ -220,7 +220,7 @@ def test_run_router_streams(tmp_path): data_key="det1", root=str(tmp_path), resource_path="test_streams", - resource_kwargs={}, + parameters={}, ) docs.append(("stream_resource", stream_resource_doc)) datum_doc = compose_stream_datum( From c98cefeda8db286817b0426e1aff482e1782e499 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 08:51:25 -0400 Subject: [PATCH 05/17] REV: revert changes to Resource, only modify Stream Resource --- docs/user/explanations/data-model.rst | 8 ++++-- docs/user/explanations/external.rst | 36 ++++++++++++++++++++---- event_model/__init__.py | 33 ++++++++++++++-------- event_model/documents/resource.py | 12 ++++++-- event_model/documents/stream_resource.py | 2 +- event_model/schemas/stream_resource.json | 6 ++-- 6 files changed, 70 insertions(+), 27 deletions(-) diff --git a/docs/user/explanations/data-model.rst b/docs/user/explanations/data-model.rst index 21fb3ec6..985ae497 100644 --- a/docs/user/explanations/data-model.rst +++ b/docs/user/explanations/data-model.rst @@ -390,7 +390,8 @@ Minimal nontrivial valid example: .. code-block:: python # 'resource' document - {'parameters': {}, + {'path_semantics': 'posix', + 'resource_kwargs': {}, 'resource_path': '/local/path/subdirectory/data_file', 'root': '/local/path/', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd', @@ -405,7 +406,8 @@ Typical example: {'spec': 'AD_HDF5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', - 'parameters': {'frame_per_point': 10}, + 'resource_kwargs': {'frame_per_point': 10}, + 'path_semantics': 'posix', 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} @@ -481,7 +483,7 @@ Typical example: # 'Stream Resource' document {'data_key': 'detector_1', - 'spec': 'AD_HDF5', + 'mimetype': 'application/x-hdf5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', 'parameters': {'frame_per_point': 1}, diff --git a/docs/user/explanations/external.rst b/docs/user/explanations/external.rst index 0c949afe..ac39f3a3 100644 --- a/docs/user/explanations/external.rst +++ b/docs/user/explanations/external.rst @@ -95,7 +95,8 @@ contains path-related details. 'spec': 'AD_HDF5', 'root': '/GPFS/DATA/Andor/', 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', - 'parameters': {'frame_per_point': 10}, + 'resource_kwargs': {'frame_per_point': 10}, + 'path_semantics': 'posix', 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} @@ -105,9 +106,32 @@ context-dependent (depending on what system you are accessing the data from) and subject to change (if the data is moved over time). The ``spec`` gives us a hint about the format of this asset, whether it be a -file, multiple files, or something more specialized. The ``parameters`` +file, multiple files, or something more specialized. The ``resource_kwargs`` provide any additional parameters for reading it. + .. code:: python + + # 'Stream Resource' document + {'uid': 'aa10035d-1d2b-41d9-97e6-03e3fe62fa6c', + 'mimetype': 'application/x-hdf5', + 'root': '/GPFS/DATA/Andor/', + 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', + 'parameters': {'frame_per_point': 10}, + 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', + 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} + +The ``resource_path`` is a relative path, all of which is semantic and should +usually not change during the lifecycle of this asset. The ``root`` is more +context-dependent (depending on what system you are accessing the data from) +and subject to change (if the data is moved over time). + +The ``mimetype`` is a recognized standard way to specify the I/O procedures to +read the asset. It gives us a hint about the format of this asset, whether it +be a file, multiple files, or something more specialized. We support standard +mimetypes, such as `image/tiff`, as well as custom ones, e.g. +`application/x-hdf5-smwr-slice`. The ``parameters`` provide any additional +parameters for reading the asset. + Handlers ======== @@ -120,7 +144,7 @@ Handler Interface A 'handler class' may be any callable with the signature:: - handler_class(full_path, **parameters) + handler_class(full_path, **resource_kwargs) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -134,8 +158,8 @@ typically implemented using a class that implements ``__init__`` and .. code:: python class MyHandler: - def __init__(self, path, **parameters): - # Consume the path information and the 'parameters' kwargs from the + def __init__(self, path, **resource_kwargs): + # Consume the path information and the 'resource_kwargs' from the # Resource. Typically stashes some state and/or opens file(s). ... @@ -150,7 +174,7 @@ But in general it may be any callable-that-returns-a-callable. .. code:: python - def handler(path, **parameters): + def handler(path, **resource_kwargs): def f(**datum_kwargs): return some_array_like return f diff --git a/event_model/__init__.py b/event_model/__init__.py index a71dbce2..3ce2f39c 100644 --- a/event_model/__init__.py +++ b/event_model/__init__.py @@ -538,7 +538,7 @@ class Filler(DocumentRouter): A 'handler class' may be any callable with the signature:: - handler_class(full_path, **parameters) + handler_class(full_path, **resource_kwargs) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -1041,7 +1041,7 @@ def get_handler(self, resource: Resource) -> Any: handler = _attempt_with_retries( func=handler_class, args=(resource_path,), - kwargs=resource["parameters"], + kwargs=resource["resource_kwargs"], intervals=[0] + self.retry_intervals, error_to_catch=IOError, error_to_raise=error_to_raise, @@ -1422,7 +1422,7 @@ class RunRouter(DocumentRouter): A 'handler class' may be any callable with the signature:: - handler_class(full_path, **parameters) + handler_class(full_path, **resource_kwargs) It is expected to return an object, a 'handler instance', which is also callable and has the following signature:: @@ -1899,6 +1899,13 @@ def __iter__(self) -> Iterator: ) +PATH_SEMANTICS: Dict[str, Literal["posix", "windows"]] = { + "posix": "posix", + "nt": "windows", +} +default_path_semantics: Literal["posix", "windows"] = PATH_SEMANTICS[os.name] + + @dataclass class ComposeResource: start: Optional[RunStart] @@ -1908,7 +1915,8 @@ def __call__( spec: str, root: str, resource_path: str, - parameters: Dict[str, Any], + resource_kwargs: Dict[str, Any], + path_semantics: Literal["posix", "windows"] = default_path_semantics, uid: Optional[str] = None, validate: bool = True, ) -> ComposeResourceBundle: @@ -1916,10 +1924,11 @@ def __call__( uid = str(uuid.uuid4()) doc = Resource( + path_semantics=path_semantics, uid=uid, spec=spec, root=root, - parameters=parameters, + resource_kwargs=resource_kwargs, resource_path=resource_path, ) @@ -1942,7 +1951,8 @@ def compose_resource( spec: str, root: str, resource_path: str, - parameters: Dict[str, Any], + resource_kwargs: Dict[str, Any], + path_semantics: Literal["posix", "windows"] = default_path_semantics, start: Optional[RunStart] = None, uid: Optional[str] = None, validate: bool = True, @@ -1954,7 +1964,8 @@ def compose_resource( spec, root, resource_path, - parameters, + resource_kwargs, + path_semantics=path_semantics, uid=uid, validate=validate, ) @@ -2036,7 +2047,7 @@ class ComposeStreamResource: def __call__( self, - spec: str, + mimetype: str, root: str, resource_path: str, data_key: str, @@ -2050,7 +2061,7 @@ def __call__( doc = StreamResource( uid=uid, data_key=data_key, - spec=spec, + mimetype=mimetype, root=root, resource_path=resource_path, parameters=parameters, @@ -2073,7 +2084,7 @@ def __call__( def compose_stream_resource( *, - spec: str, + mimetype: str, root: str, resource_path: str, data_key: str, @@ -2086,7 +2097,7 @@ def compose_stream_resource( Here for backwards compatibility, the Compose class is prefered. """ return ComposeStreamResource(start=start)( - spec, + mimetype, root, resource_path, data_key, diff --git a/event_model/documents/resource.py b/event_model/documents/resource.py index fb9227ba..882ceb69 100644 --- a/event_model/documents/resource.py +++ b/event_model/documents/resource.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from typing_extensions import Annotated, NotRequired, TypedDict +from typing_extensions import Annotated, Literal, NotRequired, TypedDict from .generate.type_wrapper import Field, add_extra_schema @@ -18,10 +18,10 @@ class PartialResource(TypedDict): resource_path: Annotated[ str, Field(description="Filepath or URI for locating this resource") ] - parameters: Annotated[ + resource_kwargs: Annotated[ Dict[str, Any], Field( - description="Additional keyword arguments to pass to the Handler to read a Resource" + description="Additional argument to pass to the Handler to read a Resource" ), ] @@ -43,6 +43,12 @@ class Resource(PartialResource): externally-stored data """ + path_semantics: NotRequired[ + Annotated[ + Literal["posix", "windows"], + Field(description="Rules for joining paths"), + ] + ] run_start: NotRequired[ Annotated[ str, diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index 4dd1f03b..30c48067 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -46,7 +46,7 @@ class StreamResource(TypedDict): ), ] ] - spec: Annotated[ + mimetype: Annotated[ str, Field( description="String identifying the format/type of this Stream Resource, " diff --git a/event_model/schemas/stream_resource.json b/event_model/schemas/stream_resource.json index 43e8ac23..f2a64cfd 100644 --- a/event_model/schemas/stream_resource.json +++ b/event_model/schemas/stream_resource.json @@ -28,8 +28,8 @@ "description": "Globally unique ID to the run_start document this Stream Resource is associated with.", "type": "string" }, - "spec": { - "title": "Spec", + "mimetype": { + "title": "Mimetype", "description": "String identifying the format/type of this Stream Resource, used to identify a compatible Handler", "type": "string" }, @@ -44,7 +44,7 @@ "parameters", "resource_path", "root", - "spec", + "mimetype", "uid" ], "additionalProperties": false From 03445b25c1f118610a1510b083fad11c4f458a32 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 09:25:11 -0400 Subject: [PATCH 06/17] BUG: fixed updated schemas --- event_model/schemas/resource.json | 17 +++++++++++++---- event_model/schemas/stream_resource.json | 12 ++++++------ event_model/tests/test_filler.py | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/event_model/schemas/resource.json b/event_model/schemas/resource.json index 7df1b449..ccbb4924 100644 --- a/event_model/schemas/resource.json +++ b/event_model/schemas/resource.json @@ -3,9 +3,18 @@ "description": "Document to reference a collection (e.g. file or group of files) of externally-stored data", "type": "object", "properties": { - "parameters": { - "title": "Parameters", - "description": "Additional keyword arguments to pass to the Handler to read a Resource", + "path_semantics": { + "title": "Path Semantics", + "description": "Rules for joining paths", + "type": "string", + "enum": [ + "posix", + "windows" + ] + }, + "resource_kwargs": { + "title": "Resource Kwargs", + "description": "Additional argument to pass to the Handler to read a Resource", "type": "object" }, "resource_path": { @@ -35,7 +44,7 @@ } }, "required": [ - "parameters", + "resource_kwargs", "resource_path", "root", "spec", diff --git a/event_model/schemas/stream_resource.json b/event_model/schemas/stream_resource.json index f2a64cfd..c673b1f2 100644 --- a/event_model/schemas/stream_resource.json +++ b/event_model/schemas/stream_resource.json @@ -8,6 +8,11 @@ "description": "A string to show which data_key of the Descriptor are being streamed", "type": "string" }, + "mimetype": { + "title": "Mimetype", + "description": "String identifying the format/type of this Stream Resource, used to identify a compatible Handler", + "type": "string" + }, "parameters": { "title": "Parameters", "description": "Additional keyword arguments to pass to the Handler to read a Stream Resource", @@ -28,11 +33,6 @@ "description": "Globally unique ID to the run_start document this Stream Resource is associated with.", "type": "string" }, - "mimetype": { - "title": "Mimetype", - "description": "String identifying the format/type of this Stream Resource, used to identify a compatible Handler", - "type": "string" - }, "uid": { "title": "Uid", "description": "Globally unique identifier for this Stream Resource", @@ -41,10 +41,10 @@ }, "required": [ "data_key", + "mimetype", "parameters", "resource_path", "root", - "mimetype", "uid" ], "additionalProperties": false diff --git a/event_model/tests/test_filler.py b/event_model/tests/test_filler.py index c1d6c660..369fdabd 100644 --- a/event_model/tests/test_filler.py +++ b/event_model/tests/test_filler.py @@ -28,7 +28,7 @@ spec="DUMMY", root=str(path_root), resource_path="stack.tiff", - parameters={"a": 1, "b": 2}, + resource_kwargs={"a": 1, "b": 2}, ) datum_doc = res_bundle.compose_datum(datum_kwargs={"c": 3, "d": 4}) raw_event = desc_bundle.compose_event( From 3d550dfa45e7b4e3d3921592cdb9ed2fd9c3e40d Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 09:40:21 -0400 Subject: [PATCH 07/17] STY: fix formatting --- event_model/tests/test_em.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index 5c6babce..a1bc5fb4 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -1020,6 +1020,4 @@ def test_array_like(): def test_resource_start_optional(): - event_model.compose_resource( - spec="TEST", root="/", resource_path="", parameters={} - ) + event_model.compose_resource(spec="TEST", root="/", resource_path="", parameters={}) From a2939a7faf13a083ed13df34343b298814d5b25d Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 09:55:18 -0400 Subject: [PATCH 08/17] BUG: fix tests --- event_model/tests/test_em.py | 22 ++++++++++++---------- event_model/tests/test_run_router.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index a1bc5fb4..5b3d5fc8 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -71,7 +71,7 @@ def test_compose_run(): assert bundle.compose_event is compose_event assert bundle.compose_event_page is compose_event_page bundle = compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) resource_doc, compose_datum, compose_datum_page = bundle assert bundle.resource_doc is resource_doc @@ -107,7 +107,7 @@ def test_compose_stream_resource(tmp_path): compose_stream_resource = bundle.compose_stream_resource assert bundle.compose_stream_resource is compose_stream_resource bundle = compose_stream_resource( - spec="TIFF_STREAM", + mimetype="application/x-hdf5", root=str(tmp_path), data_key="det1", resource_path="test_streams", @@ -134,7 +134,7 @@ def test_round_trip_pagination(): name="primary", ) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -244,7 +244,7 @@ def test_bulk_events_to_event_page(tmp_path): path_root = str(tmp_path) res_bundle = run_bundle.compose_resource( - spec="TIFF", root=path_root, resource_path="stack.tiff", parameters={} + spec="TIFF", root=path_root, resource_path="stack.tiff", resource_kwargs={} ) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum_doc2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -315,7 +315,7 @@ def test_sanitize_doc(): def test_bulk_datum_to_datum_page(): run_bundle = event_model.compose_run() res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) datum1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) datum2 = res_bundle.compose_datum(datum_kwargs={"slice": 10}) @@ -353,7 +353,7 @@ def test_document_router_smoke_test(): ) dr("descriptor", desc_bundle_baseline.descriptor_doc) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) dr("resource", res_bundle.resource_doc) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -388,7 +388,7 @@ def test_document_router_streams_smoke_test(tmp_path): start = run_bundle.start_doc dr("start", start) stream_resource_doc, compose_stream_datum = compose_stream_resource( - spec="TIFF_STREAM", + mimetype="application/x-hdf5", data_key="det1", root=str(tmp_path), resource_path="test_streams", @@ -425,7 +425,7 @@ def test_document_router_with_validation(): ) dr("descriptor", desc_bundle_baseline.descriptor_doc, validate=True) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) dr("resource", res_bundle.resource_doc, validate=True) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -762,7 +762,7 @@ def test_single_run_document_router(): ) sr("descriptor", desc_bundle_baseline.descriptor_doc) res_bundle = run_bundle.compose_resource( - spec="TIFF", root="/tmp", resource_path="stack.tiff", parameters={} + spec="TIFF", root="/tmp", resource_path="stack.tiff", resource_kwargs={} ) sr("resource", res_bundle.resource_doc) datum_doc1 = res_bundle.compose_datum(datum_kwargs={"slice": 5}) @@ -1020,4 +1020,6 @@ def test_array_like(): def test_resource_start_optional(): - event_model.compose_resource(spec="TEST", root="/", resource_path="", parameters={}) + event_model.compose_resource( + spec="TEST", root="/", resource_path="", resource_kwargs={} + ) diff --git a/event_model/tests/test_run_router.py b/event_model/tests/test_run_router.py index ef78ca06..f29832e9 100644 --- a/event_model/tests/test_run_router.py +++ b/event_model/tests/test_run_router.py @@ -33,7 +33,7 @@ def test_run_router(tmp_path): baseline_descriptor_doc, compose_baseline_event, compose_event_page = bundle docs.append(("descriptor", baseline_descriptor_doc)) bundle = compose_resource( - spec="TIFF", root=str(tmp_path), resource_path="stack.tiff", parameters={} + spec="TIFF", root=str(tmp_path), resource_path="stack.tiff", resource_kwargs={} ) resource_doc, compose_datum, compose_datum_page = bundle docs.append(("resource", resource_doc)) @@ -216,7 +216,7 @@ def test_run_router_streams(tmp_path): ) docs.append(("start", start_doc)) stream_resource_doc, compose_stream_datum = compose_stream_resource( - spec="TIFF_STREAM", + mimetype="application/x-hdf5", data_key="det1", root=str(tmp_path), resource_path="test_streams", From 52820e5c226fab4b70243a6597e0395501770e52 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 10:17:32 -0400 Subject: [PATCH 09/17] ENH: replaced root/resource_path with uri --- docs/user/explanations/external.rst | 13 +++++++------ event_model/__init__.py | 12 ++++-------- event_model/documents/stream_resource.py | 10 ++-------- event_model/schemas/stream_resource.json | 20 +++++++------------- event_model/tests/test_em.py | 6 ++---- event_model/tests/test_run_router.py | 3 +-- 6 files changed, 23 insertions(+), 41 deletions(-) diff --git a/docs/user/explanations/external.rst b/docs/user/explanations/external.rst index ac39f3a3..b47b407e 100644 --- a/docs/user/explanations/external.rst +++ b/docs/user/explanations/external.rst @@ -114,16 +114,17 @@ provide any additional parameters for reading it. # 'Stream Resource' document {'uid': 'aa10035d-1d2b-41d9-97e6-03e3fe62fa6c', 'mimetype': 'application/x-hdf5', - 'root': '/GPFS/DATA/Andor/', - 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', + 'uri': 'file://localhost/{path}/GPFS/DATA/Andor/2020/01/03/8ff08ff9.h5', 'parameters': {'frame_per_point': 10}, 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'} -The ``resource_path`` is a relative path, all of which is semantic and should -usually not change during the lifecycle of this asset. The ``root`` is more -context-dependent (depending on what system you are accessing the data from) -and subject to change (if the data is moved over time). +The ``uri`` specifies the location of the data. It may be a path on the local +filesystem, `file://localhost/{path}`, a path on a shared filesystem +`file://{host}/{path}`, to be remapped at read time via local mount config, +or a non-file-based resource like `s3://...`. The `{path}` part of the `uri` +is typically a relative path, all of which is semantic and should usually not +change during the lifecycle of this asset. The ``mimetype`` is a recognized standard way to specify the I/O procedures to read the asset. It gives us a hint about the format of this asset, whether it diff --git a/event_model/__init__.py b/event_model/__init__.py index 3ce2f39c..038bf59e 100644 --- a/event_model/__init__.py +++ b/event_model/__init__.py @@ -2048,8 +2048,7 @@ class ComposeStreamResource: def __call__( self, mimetype: str, - root: str, - resource_path: str, + uri: str, data_key: str, parameters: Dict[str, Any], uid: Optional[str] = None, @@ -2062,8 +2061,7 @@ def __call__( uid=uid, data_key=data_key, mimetype=mimetype, - root=root, - resource_path=resource_path, + uri=uri, parameters=parameters, ) @@ -2085,8 +2083,7 @@ def __call__( def compose_stream_resource( *, mimetype: str, - root: str, - resource_path: str, + uri: str, data_key: str, parameters: Dict[str, Any], start: Optional[RunStart] = None, @@ -2098,8 +2095,7 @@ def compose_stream_resource( """ return ComposeStreamResource(start=start)( mimetype, - root, - resource_path, + uri, data_key, parameters, uid=uid, diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index 30c48067..7493b421 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -28,14 +28,8 @@ class StreamResource(TypedDict): "Stream Resource", ), ] - resource_path: Annotated[ - str, Field(description="Filepath or URI for locating this resource") - ] - root: Annotated[ - str, - Field( - description="Subset of resource_path that is a local detail, not semantic." - ), + uri: Annotated[ + str, Field(description="URI for locating this resource") ] run_start: NotRequired[ Annotated[ diff --git a/event_model/schemas/stream_resource.json b/event_model/schemas/stream_resource.json index c673b1f2..3eb93b23 100644 --- a/event_model/schemas/stream_resource.json +++ b/event_model/schemas/stream_resource.json @@ -18,16 +18,6 @@ "description": "Additional keyword arguments to pass to the Handler to read a Stream Resource", "type": "object" }, - "resource_path": { - "title": "Resource Path", - "description": "Filepath or URI for locating this resource", - "type": "string" - }, - "root": { - "title": "Root", - "description": "Subset of resource_path that is a local detail, not semantic.", - "type": "string" - }, "run_start": { "title": "Run Start", "description": "Globally unique ID to the run_start document this Stream Resource is associated with.", @@ -37,15 +27,19 @@ "title": "Uid", "description": "Globally unique identifier for this Stream Resource", "type": "string" + }, + "uri": { + "title": "Uri", + "description": "URI for locating this resource", + "type": "string" } }, "required": [ "data_key", "mimetype", "parameters", - "resource_path", - "root", - "uid" + "uid", + "uri" ], "additionalProperties": false } \ No newline at end of file diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index 5b3d5fc8..17aa6ea5 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -108,9 +108,8 @@ def test_compose_stream_resource(tmp_path): assert bundle.compose_stream_resource is compose_stream_resource bundle = compose_stream_resource( mimetype="application/x-hdf5", - root=str(tmp_path), + uri="file://" + str(tmp_path) + "/test_streams", data_key="det1", - resource_path="test_streams", parameters={}, ) resource_doc, compose_stream_datum = bundle @@ -390,8 +389,7 @@ def test_document_router_streams_smoke_test(tmp_path): stream_resource_doc, compose_stream_datum = compose_stream_resource( mimetype="application/x-hdf5", data_key="det1", - root=str(tmp_path), - resource_path="test_streams", + uri="file://" + str(tmp_path) + "/test_streams", parameters={}, ) dr("stream_resource", stream_resource_doc) diff --git a/event_model/tests/test_run_router.py b/event_model/tests/test_run_router.py index f29832e9..f02e1108 100644 --- a/event_model/tests/test_run_router.py +++ b/event_model/tests/test_run_router.py @@ -218,8 +218,7 @@ def test_run_router_streams(tmp_path): stream_resource_doc, compose_stream_datum = compose_stream_resource( mimetype="application/x-hdf5", data_key="det1", - root=str(tmp_path), - resource_path="test_streams", + uri="file://" + str(tmp_path) + "/test_streams", parameters={}, ) docs.append(("stream_resource", stream_resource_doc)) From 1613ea56228848dfb2551bba180609ec33326b78 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 10:19:14 -0400 Subject: [PATCH 10/17] ENH: replaced root/resource_path with uri --- event_model/documents/stream_resource.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/event_model/documents/stream_resource.py b/event_model/documents/stream_resource.py index 7493b421..e675e581 100644 --- a/event_model/documents/stream_resource.py +++ b/event_model/documents/stream_resource.py @@ -28,9 +28,7 @@ class StreamResource(TypedDict): "Stream Resource", ), ] - uri: Annotated[ - str, Field(description="URI for locating this resource") - ] + uri: Annotated[str, Field(description="URI for locating this resource")] run_start: NotRequired[ Annotated[ str, From b8cde2443f8c46bf2b057dcb01521634ed7174a9 Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 11:57:53 -0400 Subject: [PATCH 11/17] Revert to image/tiff mimetype Co-authored-by: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> --- event_model/tests/test_em.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index 17aa6ea5..5c075045 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -107,7 +107,7 @@ def test_compose_stream_resource(tmp_path): compose_stream_resource = bundle.compose_stream_resource assert bundle.compose_stream_resource is compose_stream_resource bundle = compose_stream_resource( - mimetype="application/x-hdf5", + mimetype="image/tiff", uri="file://" + str(tmp_path) + "/test_streams", data_key="det1", parameters={}, From 7a59ad716afc6daf6b7c5ef4737c974388406dd9 Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 11:58:05 -0400 Subject: [PATCH 12/17] Revert to image/tiff mimetype Co-authored-by: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> --- event_model/tests/test_run_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_run_router.py b/event_model/tests/test_run_router.py index f02e1108..a6dbf541 100644 --- a/event_model/tests/test_run_router.py +++ b/event_model/tests/test_run_router.py @@ -216,7 +216,7 @@ def test_run_router_streams(tmp_path): ) docs.append(("start", start_doc)) stream_resource_doc, compose_stream_datum = compose_stream_resource( - mimetype="application/x-hdf5", + mimetype="image/tiff", data_key="det1", uri="file://" + str(tmp_path) + "/test_streams", parameters={}, From f0677d2f4c6eea0061fa5dbf60b7833c44c62130 Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 11:58:14 -0400 Subject: [PATCH 13/17] Revert to image/tiff mimetype Co-authored-by: DiamondJoseph <53935796+DiamondJoseph@users.noreply.github.com> --- event_model/tests/test_em.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index 5c075045..8e7b4bae 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -387,7 +387,7 @@ def test_document_router_streams_smoke_test(tmp_path): start = run_bundle.start_doc dr("start", start) stream_resource_doc, compose_stream_datum = compose_stream_resource( - mimetype="application/x-hdf5", + mimetype="image/tiff", data_key="det1", uri="file://" + str(tmp_path) + "/test_streams", parameters={}, From 2ca81141b19776d9bf51b1c6a36939c5e2b2f3e9 Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 14:41:07 -0400 Subject: [PATCH 14/17] Add localhost to uri Co-authored-by: Dan Allan --- event_model/tests/test_run_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_run_router.py b/event_model/tests/test_run_router.py index a6dbf541..b25b6391 100644 --- a/event_model/tests/test_run_router.py +++ b/event_model/tests/test_run_router.py @@ -218,7 +218,7 @@ def test_run_router_streams(tmp_path): stream_resource_doc, compose_stream_datum = compose_stream_resource( mimetype="image/tiff", data_key="det1", - uri="file://" + str(tmp_path) + "/test_streams", + uri="file://localhost" + str(tmp_path) + "/test_streams", parameters={}, ) docs.append(("stream_resource", stream_resource_doc)) From 8956fd2fa2ce6a7f67bc498b79be48d81ed946fb Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 14:55:00 -0400 Subject: [PATCH 15/17] Add localhost to uri Co-authored-by: Dan Allan --- event_model/tests/test_em.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index 8e7b4bae..b5b3a3b7 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -389,7 +389,7 @@ def test_document_router_streams_smoke_test(tmp_path): stream_resource_doc, compose_stream_datum = compose_stream_resource( mimetype="image/tiff", data_key="det1", - uri="file://" + str(tmp_path) + "/test_streams", + uri="file://localhost" + str(tmp_path) + "/test_streams", parameters={}, ) dr("stream_resource", stream_resource_doc) From 81ace7ecc8c0340f96a0be6c511c5a60515eaefb Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 2 Apr 2024 14:55:08 -0400 Subject: [PATCH 16/17] Add localhost to uri Co-authored-by: Dan Allan --- event_model/tests/test_em.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event_model/tests/test_em.py b/event_model/tests/test_em.py index b5b3a3b7..15e0fd81 100644 --- a/event_model/tests/test_em.py +++ b/event_model/tests/test_em.py @@ -108,7 +108,7 @@ def test_compose_stream_resource(tmp_path): assert bundle.compose_stream_resource is compose_stream_resource bundle = compose_stream_resource( mimetype="image/tiff", - uri="file://" + str(tmp_path) + "/test_streams", + uri="file://localhost" + str(tmp_path) + "/test_streams", data_key="det1", parameters={}, ) From b9b058659995cb5fe930d1f0f22a13f0f10e0a06 Mon Sep 17 00:00:00 2001 From: Eugene M Date: Tue, 2 Apr 2024 14:59:55 -0400 Subject: [PATCH 17/17] FIX: replace root/resource_path with uri --- docs/user/explanations/data-model.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/user/explanations/data-model.rst b/docs/user/explanations/data-model.rst index 985ae497..d5efeb7c 100644 --- a/docs/user/explanations/data-model.rst +++ b/docs/user/explanations/data-model.rst @@ -484,8 +484,7 @@ Typical example: # 'Stream Resource' document {'data_key': 'detector_1', 'mimetype': 'application/x-hdf5', - 'root': '/GPFS/DATA/Andor/', - 'resource_path': '2020/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', + 'uri': 'file://localhost/GPFS/DATA/Andor/01/03/8ff08ff9-a2bf-48c3-8ff3-dcac0f309d7d.h5', 'parameters': {'frame_per_point': 1}, 'uid': '3b300e6f-b431-4750-a635-5630d15c81a8', 'run_start': '10bf6945-4afd-43ca-af36-6ad8f3540bcd'}