From ba01c439131e1a0515ad99d962992859d6283ee6 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 22 Oct 2024 14:50:43 -0400 Subject: [PATCH 01/14] add deps to requirements-test --- requirements-test.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements-test.txt b/requirements-test.txt index 469edab16..d83f8614c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,15 +1,19 @@ area-detector-handlers bluesky +boltons codecov coverage +doct flake8 glue-core <1.18 glueviz matplotlib mongomock +mongoquery msgpack >=1.0.0 numpy >=1.16.0 # for astropy (via glueviz) ophyd +pims pyqt5 !=5.14.1 pytest >=4.4,!=5.4.0 pytest-rerunfailures @@ -18,5 +22,6 @@ suitcase-jsonl >=0.1.0b2 suitcase-mongo >=0.5.0 suitcase-msgpack >=0.2.2 tiled[all] >=0.1.0b9 +tzlocal ujson vcrpy From 10ca0ed0d8de097ce1afdcba08627d7a97d5a1c0 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 22 Oct 2024 15:23:40 -0400 Subject: [PATCH 02/14] starting to add more tests --- databroker/tests/test_validate_shape.py | 101 +++++++++++++++++++----- 1 file changed, 82 insertions(+), 19 deletions(-) diff --git a/databroker/tests/test_validate_shape.py b/databroker/tests/test_validate_shape.py index df563e384..edc2bcfa0 100644 --- a/databroker/tests/test_validate_shape.py +++ b/databroker/tests/test_validate_shape.py @@ -10,42 +10,81 @@ import pytest -def test_validate_shape(tmpdir): - # custom_validate_shape will mutate this to show it has been called - shapes = [] - - def custom_validate_shape(key, data, expected_shape): - shapes.append(expected_shape) - return data +@pytest.mark.parametrize( + "shape,expected_shape", + [ + ((10,), (11,)), # Short by 1, 1d. + ((10, 20), (10, 21)), # Short by 1, 2d. + ((10, 20, 30), (10, 21, 30)), # Short by 1, 3d. + ((10, 20, 30), (10, 20, 31)), # Short by 1, 3d. + ((10, 20), (10, 19)), # Too-big by 1, 2d. + ((20, 20, 20, 20), (20, 21, 20, 22)), # 4d example. + # TODO: Need to add some ragged test cases. + ], +) +def test_padding(tmpdir, shape, expected_shape): + adapter = MongoAdapter.from_mongomock() - adapter = MongoAdapter.from_mongomock(validate_shape=custom_validate_shape) + direct_img = DirectImage( + func=lambda: np.array(np.ones(shape)), name="direct", labels={"detectors"} + ) + direct_img.img.name = "img" with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: client = from_context(context) def post_document(name, doc): + if name == "descriptor": + doc["data_keys"]["img"]["shape"] = expected_shape + client.post_document(name, doc) RE = RunEngine() RE.subscribe(post_document) - (uid,) = RE(count([img])) - assert not shapes - client[uid]["primary"]["data"]["img"][:] - assert shapes + (uid,) = RE(count([direct_img])) + assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape @pytest.mark.parametrize( "shape,expected_shape", [ - ((10,), (11,)), - ((10, 20), (10, 21)), - ((10, 20), (10, 19)), - ((10, 20, 30), (10, 21, 30)), - ((10, 20, 30), (10, 20, 31)), - ((20, 20, 20, 20), (20, 21, 20, 22)), + # TODO: Need figure out test cases. + # ], ) -def test_padding(tmpdir, shape, expected_shape): +def test_default_chunking(tmpdir, shape, expected_shape): + adapter = MongoAdapter.from_mongomock() + + direct_img = DirectImage( + func=lambda: np.array(np.ones(shape)), name="direct", labels={"detectors"} + ) + direct_img.img.name = "img" + + with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: + client = from_context(context) + + def post_document(name, doc): + if name == "descriptor": + doc["data_keys"]["img"]["shape"] = expected_shape + + client.post_document(name, doc) + + RE = RunEngine() + RE.subscribe(post_document) + (uid,) = RE(count([direct_img])) + assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape + + +@pytest.mark.parametrize( + "shape,chunks", + [ + # TODO: Need figure out test cases. + # Accepts 'auto'. + # suggested_chunks don’t line up perfectly with the data size. + # supports chunk_shape in addition to chunks_list. + ], +) +def test_custom_chunking(tmpdir, shape, expected_shape): adapter = MongoAdapter.from_mongomock() direct_img = DirectImage( @@ -99,3 +138,27 @@ def post_document(name, doc): (uid,) = RE(count([direct_img])) with pytest.raises(BadShapeMetadata): client[uid]["primary"]["data"]["img"][:] + + +def test_custom_validate_shape(tmpdir): + # custom_validate_shape will mutate this to show it has been called + shapes = [] + + def custom_validate_shape(key, data, expected_shape): + shapes.append(expected_shape) + return data + + adapter = MongoAdapter.from_mongomock(validate_shape=custom_validate_shape) + + with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: + client = from_context(context) + + def post_document(name, doc): + client.post_document(name, doc) + + RE = RunEngine() + RE.subscribe(post_document) + (uid,) = RE(count([img])) + assert not shapes + client[uid]["primary"]["data"]["img"][:] + assert shapes From 5485fb49e9979348a1187b08824ffa143eada5b1 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 22 Oct 2024 15:48:52 -0400 Subject: [PATCH 03/14] touch ups --- databroker/tests/test_validate_shape.py | 42 ++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/databroker/tests/test_validate_shape.py b/databroker/tests/test_validate_shape.py index edc2bcfa0..fa89421b2 100644 --- a/databroker/tests/test_validate_shape.py +++ b/databroker/tests/test_validate_shape.py @@ -48,8 +48,36 @@ def post_document(name, doc): @pytest.mark.parametrize( "shape,expected_shape", [ - # TODO: Need figure out test cases. - # + # TODO: Need to add some ragged test cases. + ], +) +def test_ragged_padding(tmpdir, shape, expected_shape): + adapter = MongoAdapter.from_mongomock() + + direct_img = DirectImage( + func=lambda: np.array(np.ones(shape)), name="direct", labels={"detectors"} + ) + direct_img.img.name = "img" + + with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: + client = from_context(context) + + def post_document(name, doc): + if name == "descriptor": + doc["data_keys"]["img"]["shape"] = expected_shape + + client.post_document(name, doc) + + RE = RunEngine() + RE.subscribe(post_document) + (uid,) = RE(count([direct_img])) + assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape + + +@pytest.mark.parametrize( + "shape,expected_chunks", + [ + # default chunks shouldn't span files, this will fix read-timouts. ], ) def test_default_chunking(tmpdir, shape, expected_shape): @@ -110,13 +138,13 @@ def post_document(name, doc): @pytest.mark.parametrize( "shape,expected_shape", [ - ((10,), (11, 12)), - ((10, 20), (10, 200)), - ((20, 20, 20, 20), (20, 21, 20, 200)), - ((10, 20), (5, 20)), + ((10,), (11, 12)), # Different number of dimensions. + ((10, 20), (10, 200)), # Dimension sizes differ by more than 2. + ((20, 20, 20, 20), (20, 21, 20, 200)), # Dimension sizes differ by more than 2. + ((10, 20), (5, 20)), # Data is bigger than expected. ], ) -def test_default_validate_shape(tmpdir, shape, expected_shape): +def test_validate_shape_exceptions(tmpdir, shape, expected_shape): adapter = MongoAdapter.from_mongomock() direct_img = DirectImage( From 8bf737f34b66df6fb15e26f457e07402586bd753 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 15:34:13 -0400 Subject: [PATCH 04/14] got the tests passing --- databroker/mongo_normalized.py | 4 +- databroker/tests/test_validate_shape.py | 98 +++++++------------------ 2 files changed, 29 insertions(+), 73 deletions(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index 3ad914510..c60b61ea3 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -160,7 +160,9 @@ def structure_from_descriptor(descriptor, sub_dict, max_seq_num, unicode_columns numpy_dtype = dtype.to_numpy_dtype() if "chunks" in field_metadata: # If the Event Descriptor tells us a preferred chunking, use that. - suggested_chunks = tuple(tuple(chunks) for chunks in field_metadata["chunks"]) + suggested_chunks = [tuple(chunk) if isinstance(chunk, list) + else chunk for chunk in field_metadata['chunks']] + print(suggested_chunks, shape) elif (0 in shape) or (numpy_dtype.itemsize == 0): # special case to avoid warning from dask suggested_chunks = shape diff --git a/databroker/tests/test_validate_shape.py b/databroker/tests/test_validate_shape.py index fa89421b2..46a3f1607 100644 --- a/databroker/tests/test_validate_shape.py +++ b/databroker/tests/test_validate_shape.py @@ -6,6 +6,7 @@ from ..mongo_normalized import MongoAdapter, BadShapeMetadata +import json import numpy as np import pytest @@ -19,7 +20,6 @@ ((10, 20, 30), (10, 20, 31)), # Short by 1, 3d. ((10, 20), (10, 19)), # Too-big by 1, 2d. ((20, 20, 20, 20), (20, 21, 20, 22)), # 4d example. - # TODO: Need to add some ragged test cases. ], ) def test_padding(tmpdir, shape, expected_shape): @@ -46,73 +46,26 @@ def post_document(name, doc): @pytest.mark.parametrize( - "shape,expected_shape", - [ - # TODO: Need to add some ragged test cases. - ], -) -def test_ragged_padding(tmpdir, shape, expected_shape): - adapter = MongoAdapter.from_mongomock() - - direct_img = DirectImage( - func=lambda: np.array(np.ones(shape)), name="direct", labels={"detectors"} - ) - direct_img.img.name = "img" - - with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: - client = from_context(context) - - def post_document(name, doc): - if name == "descriptor": - doc["data_keys"]["img"]["shape"] = expected_shape - - client.post_document(name, doc) - - RE = RunEngine() - RE.subscribe(post_document) - (uid,) = RE(count([direct_img])) - assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape - - -@pytest.mark.parametrize( - "shape,expected_chunks", - [ - # default chunks shouldn't span files, this will fix read-timouts. - ], -) -def test_default_chunking(tmpdir, shape, expected_shape): - adapter = MongoAdapter.from_mongomock() - - direct_img = DirectImage( - func=lambda: np.array(np.ones(shape)), name="direct", labels={"detectors"} - ) - direct_img.img.name = "img" - - with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: - client = from_context(context) - - def post_document(name, doc): - if name == "descriptor": - doc["data_keys"]["img"]["shape"] = expected_shape - - client.post_document(name, doc) - - RE = RunEngine() - RE.subscribe(post_document) - (uid,) = RE(count([direct_img])) - assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape - - -@pytest.mark.parametrize( - "shape,chunks", + "chunks,shape,expected_chunks", [ - # TODO: Need figure out test cases. - # Accepts 'auto'. - # suggested_chunks don’t line up perfectly with the data size. - # supports chunk_shape in addition to chunks_list. + ([1, 2], (10,), ((1,), (2, 2, 2, 2, 2))), # 1D image + ([1, 3], (10,), ((1,), (3, 3, 3, 1))), # not evenly divisible. + ([1, 2, 2], (10, 10), ((1,), (2, 2, 2, 2, 2), (2, 2, 2, 2, 2))), # 2D + ([1, 2, -1], (10, 10), ((1,), (2, 2, 2, 2, 2), (10,))), # -1 for max size. + ([1, 2, "auto"], (10, 10), ((1,), (2, 2, 2, 2, 2), (10,))), # auto + ( + ((1,), (2, 2, 2, 2, 2), (2, 2, 2, 2, 2)), + (10, 10), + ((1,), (2, 2, 2, 2, 2), (2, 2, 2, 2, 2)), + ), # normalized chunks + ( + [1, 5, "auto", -1, 5], + (10, 10, 10, 10), + ((1,), (5, 5), (10,), (10,), (5, 5)) + ), # mixture of things. ], ) -def test_custom_chunking(tmpdir, shape, expected_shape): +def test_custom_chunking(tmpdir, chunks, shape, expected_chunks): adapter = MongoAdapter.from_mongomock() direct_img = DirectImage( @@ -121,27 +74,28 @@ def test_custom_chunking(tmpdir, shape, expected_shape): direct_img.img.name = "img" with Context.from_app(build_app(adapter), token_cache=tmpdir) as context: - client = from_context(context) + client = from_context(context, "dask") def post_document(name, doc): if name == "descriptor": - doc["data_keys"]["img"]["shape"] = expected_shape + doc["data_keys"]["img"]["chunks"] = chunks client.post_document(name, doc) RE = RunEngine() RE.subscribe(post_document) (uid,) = RE(count([direct_img])) - assert client[uid]["primary"]["data"]["img"][0].shape == expected_shape + assert client[uid]["primary"]["data"]["img"].chunks == expected_chunks + # assert client[uid]["primary"]["data"]["img"][0].shape == shape @pytest.mark.parametrize( "shape,expected_shape", [ - ((10,), (11, 12)), # Different number of dimensions. - ((10, 20), (10, 200)), # Dimension sizes differ by more than 2. - ((20, 20, 20, 20), (20, 21, 20, 200)), # Dimension sizes differ by more than 2. - ((10, 20), (5, 20)), # Data is bigger than expected. + ((10,), (11, 12)), # Different number of dimensions. + ((10, 20), (10, 200)), # Dimension sizes differ by more than 2. + ((20, 20, 20, 20), (20, 21, 20, 200)), # Dimension sizes differ by more than 2. + ((10, 20), (5, 20)), # Data is bigger than expected. ], ) def test_validate_shape_exceptions(tmpdir, shape, expected_shape): From eeeb9ab126b8f91cf44a638d5bc071bb200ca797 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 15:36:06 -0400 Subject: [PATCH 05/14] remove debug print --- databroker/mongo_normalized.py | 1 - 1 file changed, 1 deletion(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index c60b61ea3..ca12159f6 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -162,7 +162,6 @@ def structure_from_descriptor(descriptor, sub_dict, max_seq_num, unicode_columns # If the Event Descriptor tells us a preferred chunking, use that. suggested_chunks = [tuple(chunk) if isinstance(chunk, list) else chunk for chunk in field_metadata['chunks']] - print(suggested_chunks, shape) elif (0 in shape) or (numpy_dtype.itemsize == 0): # special case to avoid warning from dask suggested_chunks = shape From bb4b68cf8cb3dc3e647f09ff45bbe0d7cf850432 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 16:02:19 -0400 Subject: [PATCH 06/14] padding warning --- databroker/mongo_normalized.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index ca12159f6..9674d16e0 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -2212,7 +2212,7 @@ def default_validate_shape(key, data, expected_shape): * If number of dimensions differ, raise BadShapeMetadata * If any dimension differs by more than MAX_SIZE_DIFF, raise BadShapeMetadata. * If some dimensions are smaller than expected,, pad "right" edge of each - dimension that falls short with NaN. + dimension that falls short with zeros.. """ MAX_SIZE_DIFF = 2 if data.shape == expected_shape: @@ -2242,6 +2242,11 @@ def default_validate_shape(key, data, expected_shape): else: # margin == 0 padding.append((0, 0)) padded = numpy.pad(data, padding, "edge") + + logger.warning(f"The data.shape: {data.shape} did not match the expected_shape: " + f"{expected_shape} for key: '{key}'. This data has been zero-padded " + "to match the expected shape!") + return padded From efd3e2bc420a007ae18bf8ee071f39f239c5c253 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 16:03:24 -0400 Subject: [PATCH 07/14] remove unused import --- databroker/tests/test_validate_shape.py | 1 - 1 file changed, 1 deletion(-) diff --git a/databroker/tests/test_validate_shape.py b/databroker/tests/test_validate_shape.py index 46a3f1607..adf932541 100644 --- a/databroker/tests/test_validate_shape.py +++ b/databroker/tests/test_validate_shape.py @@ -6,7 +6,6 @@ from ..mongo_normalized import MongoAdapter, BadShapeMetadata -import json import numpy as np import pytest From 9e65597ebe9d6ec67aeff3170b74251ef4ae94b0 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 16:05:34 -0400 Subject: [PATCH 08/14] touch up --- databroker/mongo_normalized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index 9674d16e0..f28aa419c 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -2245,7 +2245,7 @@ def default_validate_shape(key, data, expected_shape): logger.warning(f"The data.shape: {data.shape} did not match the expected_shape: " f"{expected_shape} for key: '{key}'. This data has been zero-padded " - "to match the expected shape!") + "to match the expected_shape!") return padded From da83734094210c2e78960ac6e60342bae6f4d811 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 16:18:06 -0400 Subject: [PATCH 09/14] extra . --- databroker/mongo_normalized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index f28aa419c..98b90edc1 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -2212,7 +2212,7 @@ def default_validate_shape(key, data, expected_shape): * If number of dimensions differ, raise BadShapeMetadata * If any dimension differs by more than MAX_SIZE_DIFF, raise BadShapeMetadata. * If some dimensions are smaller than expected,, pad "right" edge of each - dimension that falls short with zeros.. + dimension that falls short with zeros. """ MAX_SIZE_DIFF = 2 if data.shape == expected_shape: From 23fd04a680113cc73a9bdfd1717f5640cb6f6cd0 Mon Sep 17 00:00:00 2001 From: gbischof Date: Fri, 25 Oct 2024 16:19:46 -0400 Subject: [PATCH 10/14] touch up --- databroker/tests/test_validate_shape.py | 1 - 1 file changed, 1 deletion(-) diff --git a/databroker/tests/test_validate_shape.py b/databroker/tests/test_validate_shape.py index adf932541..0f2ca4c89 100644 --- a/databroker/tests/test_validate_shape.py +++ b/databroker/tests/test_validate_shape.py @@ -85,7 +85,6 @@ def post_document(name, doc): RE.subscribe(post_document) (uid,) = RE(count([direct_img])) assert client[uid]["primary"]["data"]["img"].chunks == expected_chunks - # assert client[uid]["primary"]["data"]["img"][0].shape == shape @pytest.mark.parametrize( From df304333e639fa7db6e4a71155a1ea910ed259b3 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 29 Oct 2024 11:52:16 -0400 Subject: [PATCH 11/14] fix test_access_policy_example --- databroker/tests/conftest.py | 19 ------------------- databroker/tests/test_access_policy.py | 8 ++++---- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/databroker/tests/conftest.py b/databroker/tests/conftest.py index 7d8468bd1..0962d5c0f 100644 --- a/databroker/tests/conftest.py +++ b/databroker/tests/conftest.py @@ -114,22 +114,3 @@ def delete_dm(): @pytest.fixture(params=['scalar', 'image', 'external_image']) def detector(request, hw): return getattr(hw, SIM_DETECTORS[request.param]) - - -@pytest.fixture -def enter_password(monkeypatch): - """ - Return a context manager that overrides getpass, used like: - - >>> with enter_password(...): - ... # Run code that calls getpass.getpass(). - """ - - @contextlib.contextmanager - def f(password): - original = getpass.getpass - monkeypatch.setattr("getpass.getpass", lambda: password) - yield - monkeypatch.setattr("getpass.getpass", original) - - return f diff --git a/databroker/tests/test_access_policy.py b/databroker/tests/test_access_policy.py index 6a18a5985..3c2dd97b2 100644 --- a/databroker/tests/test_access_policy.py +++ b/databroker/tests/test_access_policy.py @@ -2,7 +2,7 @@ from bluesky.plans import count from tiled.client import Context, from_context from tiled.server.app import build_app_from_config - +from tiled._tests.utils import enter_username_password from ..mongo_normalized import MongoAdapter, SimpleAccessPolicy @@ -22,7 +22,7 @@ def __init__(self, *args, **kwargs): InstrumentedMongoAdapter.from_mongomock(access_policy=access_policy) -def test_access_policy_example(tmpdir, enter_password): +def test_access_policy_example(tmpdir): config = { "authentication": { @@ -52,8 +52,8 @@ def test_access_policy_example(tmpdir, enter_password): ], } with Context.from_app(build_app_from_config(config), token_cache=tmpdir) as context: - with enter_password("secret"): - client = from_context(context, username="alice", prompt_for_reauthentication=True) + with enter_username_password("alice", "secret"): + client = from_context(context, prompt_for_reauthentication=True) def post_document(name, doc): client.post_document(name, doc) From 56892c807d51834e719782b2eee8cbe52a8eef4b Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 29 Oct 2024 12:00:58 -0400 Subject: [PATCH 12/14] Revert "add deps to requirements-test" This reverts commit ba01c439131e1a0515ad99d962992859d6283ee6. --- requirements-test.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index d83f8614c..469edab16 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,19 +1,15 @@ area-detector-handlers bluesky -boltons codecov coverage -doct flake8 glue-core <1.18 glueviz matplotlib mongomock -mongoquery msgpack >=1.0.0 numpy >=1.16.0 # for astropy (via glueviz) ophyd -pims pyqt5 !=5.14.1 pytest >=4.4,!=5.4.0 pytest-rerunfailures @@ -22,6 +18,5 @@ suitcase-jsonl >=0.1.0b2 suitcase-mongo >=0.5.0 suitcase-msgpack >=0.2.2 tiled[all] >=0.1.0b9 -tzlocal ujson vcrpy From 86cd7fd142abd5675e8c49df258c0c784a54de88 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 29 Oct 2024 12:24:57 -0400 Subject: [PATCH 13/14] remove extra imports --- databroker/mongo_normalized.py | 2 +- databroker/tests/conftest.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index 98b90edc1..17ca49928 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -2205,7 +2205,7 @@ class BadShapeMetadata(Exception): pass -def default_validate_shape(key, data, expected_shape): +def default_validate_shape(key, data, expected_shape, uid=None): """ Check that data.shape == expected.shape. diff --git a/databroker/tests/conftest.py b/databroker/tests/conftest.py index 0962d5c0f..d7864a569 100644 --- a/databroker/tests/conftest.py +++ b/databroker/tests/conftest.py @@ -1,5 +1,3 @@ -import contextlib -import getpass import os import pytest import sys From 81c122b8db7c7e8f309c3a29ee9cbe3a6915bea4 Mon Sep 17 00:00:00 2001 From: gbischof Date: Tue, 29 Oct 2024 12:41:04 -0400 Subject: [PATCH 14/14] pass uid to default_validate_shape --- databroker/mongo_normalized.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/databroker/mongo_normalized.py b/databroker/mongo_normalized.py index 17ca49928..15d258a8f 100644 --- a/databroker/mongo_normalized.py +++ b/databroker/mongo_normalized.py @@ -4,6 +4,7 @@ import copy from datetime import datetime, timedelta import functools +import inspect import itertools import logging import os @@ -932,6 +933,9 @@ def populate_columns(keys, min_seq_num, max_seq_num): map( lambda item: self.validate_shape( key, numpy.asarray(item), expected_shape + ) if 'uid' in inspect.signature(self.validate_shape).parameters + else self.validate_shape( + key, numpy.asarray(item), expected_shape, uid=self._run.metadata()['start']['uid'] ), result[key], ) @@ -2245,7 +2249,7 @@ def default_validate_shape(key, data, expected_shape, uid=None): logger.warning(f"The data.shape: {data.shape} did not match the expected_shape: " f"{expected_shape} for key: '{key}'. This data has been zero-padded " - "to match the expected_shape!") + "to match the expected_shape! RunStart UID: {uid}") return padded