From 133a189f7d011dc37e4498b287181c9f9c043a01 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 24 Jan 2024 15:54:58 -0500 Subject: [PATCH] more refactoring related to PortalObject.compare for submitr. --- dcicutils/data_readers.py | 21 +++++++++++---------- dcicutils/misc_utils.py | 11 +++++++---- dcicutils/portal_object_utils.py | 13 +++++++++---- dcicutils/structured_data.py | 11 +++++++---- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/dcicutils/data_readers.py b/dcicutils/data_readers.py index d6c50e2c8..cb5bfb2f1 100644 --- a/dcicutils/data_readers.py +++ b/dcicutils/data_readers.py @@ -7,18 +7,19 @@ # Forward type references for type hints. Excel = Type["Excel"] +# Cell values(s) indicating property deletion. +_CELL_DELETION_VALUES = ["*delete*"] -class RowReader(abc.ABC): - # Cell values(s) indicating property deletion. - _CELL_DELETION_VALUES = ["*delete*"] +# Special cell deletion sentinel value (note make sure on deepcopy it remains the same). +class _CellDeletionSentinal(str): + def __new__(cls): + return super(_CellDeletionSentinal, cls).__new__(cls, _CELL_DELETION_VALUES[0]) + def __deepcopy__(self, memo): # noqa + return self + - # Special cell deletion sentinel value (note make sure on deepcopy it remains the same). - class _CellDeletionSentinal(object): - def __str__(self): - return RowReader._CELL_DELETION_VALUES[0] - def __deepcopy__(self, memo): # noqa - return self +class RowReader(abc.ABC): CELL_DELETION_SENTINEL = _CellDeletionSentinal() @@ -60,7 +61,7 @@ def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any] def cell_value(self, value: Optional[Any]) -> str: if value is None: return "" - elif (value := str(value).strip()) in RowReader._CELL_DELETION_VALUES: + elif (value := str(value).strip()) in _CELL_DELETION_VALUES: return RowReader.CELL_DELETION_SENTINEL else: return value diff --git a/dcicutils/misc_utils.py b/dcicutils/misc_utils.py index 922e5de5d..7c6a41951 100644 --- a/dcicutils/misc_utils.py +++ b/dcicutils/misc_utils.py @@ -1148,16 +1148,19 @@ def remove_suffix(suffix: str, text: str, required: bool = False): return text[:len(text)-len(suffix)] -def remove_empty_properties(data: Optional[Union[list, dict]]) -> None: +def remove_empty_properties(data: Optional[Union[list, dict]], isempty: Optional[Callable] = None) -> None: + def _isempty(value: Any) -> bool: # noqa + return isempty(value) if callable(isempty) else value in [None, "", {}, []] if isinstance(data, dict): for key in list(data.keys()): - if (value := data[key]) in [None, "", {}, []]: + if _isempty(value := data[key]): del data[key] else: - remove_empty_properties(value) + remove_empty_properties(value, isempty=isempty) elif isinstance(data, list): for item in data: - remove_empty_properties(item) + remove_empty_properties(item, isempty=isempty) + data[:] = [item for item in data if not _isempty(item)] class ObsoleteError(Exception): diff --git a/dcicutils/portal_object_utils.py b/dcicutils/portal_object_utils.py index 223f2204c..1e79a7627 100644 --- a/dcicutils/portal_object_utils.py +++ b/dcicutils/portal_object_utils.py @@ -11,6 +11,8 @@ class PortalObject: + _PROPERTY_DELETION_SENTINEL = RowReader.CELL_DELETION_SENTINEL + def __init__(self, portal: Portal, portal_object: dict, portal_object_type: Optional[str] = None) -> None: self._portal = portal self._data = portal_object @@ -145,7 +147,7 @@ def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict: for key in a: path = f"{_path}.{key}" if _path else key if key not in b: - if a[key] != RowReader.CELL_DELETION_SENTINEL: + if a[key] != PortalObject._PROPERTY_DELETION_SENTINEL: diffs[path] = {"value": a[key], "creating_value": True} else: diffs.update(PortalObject._compare(a[key], b[key], _path=path)) @@ -155,17 +157,20 @@ def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict: path = f"{_path or ''}#{index}" if not isinstance(a[index], dict) and not isinstance(a[index], list): if a[index] not in b: - if a[index] != RowReader.CELL_DELETION_SENTINEL: - if len(b) < index: + if a[index] != PortalObject._PROPERTY_DELETION_SENTINEL: + if index < len(b): diffs[path] = {"value": a[index], "updating_value": b[index]} else: diffs[path] = {"value": a[index], "creating_value": True} + else: + if index < len(b): + diffs[path] = {"value": b[index], "deleting_value": True} elif len(b) < index: diffs.update(PortalObject._compare(a[index], b[index], _path=path)) else: diffs[path] = {"value": a[index], "creating_value": True} elif a != b: - if a == RowReader.CELL_DELETION_SENTINEL: + if a == PortalObject._PROPERTY_DELETION_SENTINEL: diffs[_path] = {"value": b, "deleting_value": True} else: diffs[_path] = {"value": a, "updating_value": b} diff --git a/dcicutils/structured_data.py b/dcicutils/structured_data.py index 42a57860b..ed5e16809 100644 --- a/dcicutils/structured_data.py +++ b/dcicutils/structured_data.py @@ -70,10 +70,13 @@ def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None, def validate(self, force: bool = False) -> None: def data_without_deleted_properties(data: dict) -> dict: nonlocal self - if self._prune: - return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL} - else: - return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()} + def isempty(value: Any) -> bool: # noqa + if value == RowReader.CELL_DELETION_SENTINEL: + return True + return self._prune and value in [None, "", {}, []] + data = copy.deepcopy(data) + remove_empty_properties(data, isempty=isempty) + return data if self._validated and not force: return self._validated = True