more refactoring related to PortalObject.compare for submitr.

dcicutils/data_readers.py

@@ -7,18 +7,19 @@
 # Forward type references for type hints.
 Excel = Type["Excel"]
 
+# Cell values(s) indicating property deletion.
+_CELL_DELETION_VALUES = ["*delete*"]
 
-class RowReader(abc.ABC):
 
-    # Cell values(s) indicating property deletion.
-    _CELL_DELETION_VALUES = ["*delete*"]
+# Special cell deletion sentinel value (note make sure on deepcopy it remains the same).
+class _CellDeletionSentinal(str):
+    def __new__(cls):
+        return super(_CellDeletionSentinal, cls).__new__(cls, _CELL_DELETION_VALUES[0])
+    def __deepcopy__(self, memo):  # noqa
+        return self
+
 
-    # Special cell deletion sentinel value (note make sure on deepcopy it remains the same).
-    class _CellDeletionSentinal(object):
-        def __str__(self):
-            return RowReader._CELL_DELETION_VALUES[0]
-        def __deepcopy__(self, memo):  # noqa
-            return self
+class RowReader(abc.ABC):
 
     CELL_DELETION_SENTINEL = _CellDeletionSentinal()
 
@@ -60,7 +61,7 @@ def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any]
     def cell_value(self, value: Optional[Any]) -> str:
         if value is None:
             return ""
-        elif (value := str(value).strip()) in RowReader._CELL_DELETION_VALUES:
+        elif (value := str(value).strip()) in _CELL_DELETION_VALUES:
             return RowReader.CELL_DELETION_SENTINEL
         else:
             return value

dcicutils/misc_utils.py

@@ -1148,16 +1148,19 @@ def remove_suffix(suffix: str, text: str, required: bool = False):
     return text[:len(text)-len(suffix)]
 
 
-def remove_empty_properties(data: Optional[Union[list, dict]]) -> None:
+def remove_empty_properties(data: Optional[Union[list, dict]], isempty: Optional[Callable] = None) -> None:
+    def _isempty(value: Any) -> bool:  # noqa
+        return isempty(value) if callable(isempty) else value in [None, "", {}, []]
     if isinstance(data, dict):
         for key in list(data.keys()):
-            if (value := data[key]) in [None, "", {}, []]:
+            if _isempty(value := data[key]):
                 del data[key]
             else:
-                remove_empty_properties(value)
+                remove_empty_properties(value, isempty=isempty)
     elif isinstance(data, list):
         for item in data:
-            remove_empty_properties(item)
+            remove_empty_properties(item, isempty=isempty)
+        data[:] = [item for item in data if not _isempty(item)]
 
 
 class ObsoleteError(Exception):

dcicutils/portal_object_utils.py

@@ -11,6 +11,8 @@
 
 class PortalObject:
 
+    _PROPERTY_DELETION_SENTINEL = RowReader.CELL_DELETION_SENTINEL
+
     def __init__(self, portal: Portal, portal_object: dict, portal_object_type: Optional[str] = None) -> None:
         self._portal = portal
         self._data = portal_object
@@ -145,7 +147,7 @@ def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict:
             for key in a:
                 path = f"{_path}.{key}" if _path else key
                 if key not in b:
-                    if a[key] != RowReader.CELL_DELETION_SENTINEL:
+                    if a[key] != PortalObject._PROPERTY_DELETION_SENTINEL:
                         diffs[path] = {"value": a[key], "creating_value": True}
                 else:
                     diffs.update(PortalObject._compare(a[key], b[key], _path=path))
@@ -155,17 +157,20 @@ def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict:
                 path = f"{_path or ''}#{index}"
                 if not isinstance(a[index], dict) and not isinstance(a[index], list):
                     if a[index] not in b:
-                        if a[index] != RowReader.CELL_DELETION_SENTINEL:
-                            if len(b) < index:
+                        if a[index] != PortalObject._PROPERTY_DELETION_SENTINEL:
+                            if index < len(b):
                                 diffs[path] = {"value": a[index], "updating_value": b[index]}
                             else:
                                 diffs[path] = {"value": a[index], "creating_value": True}
+                        else:
+                            if index < len(b):
+                                diffs[path] = {"value": b[index], "deleting_value": True}
                 elif len(b) < index:
                     diffs.update(PortalObject._compare(a[index], b[index], _path=path))
                 else:
                     diffs[path] = {"value": a[index], "creating_value": True}
         elif a != b:
-            if a == RowReader.CELL_DELETION_SENTINEL:
+            if a == PortalObject._PROPERTY_DELETION_SENTINEL:
                 diffs[_path] = {"value": b, "deleting_value": True}
             else:
                 diffs[_path] = {"value": a, "updating_value": b}

dcicutils/structured_data.py

@@ -70,10 +70,13 @@ def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
     def validate(self, force: bool = False) -> None:
         def data_without_deleted_properties(data: dict) -> dict:
             nonlocal self
-            if self._prune:
-                return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL}
-            else:
-                return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()}
+            def isempty(value: Any) -> bool:  # noqa
+                if value == RowReader.CELL_DELETION_SENTINEL:
+                    return True
+                return self._prune and value in [None, "", {}, []]
+            data = copy.deepcopy(data)
+            remove_empty_properties(data, isempty=isempty)
+            return data
         if self._validated and not force:
             return
         self._validated = True