added autoadd support to structured_data.

CHANGELOG.rst

@@ -10,6 +10,9 @@ Change Log
 =====
 * Minor fix to misc_utils.to_integer to handle float strings.
 * Minor fix to structured_data to accumulate unique resolved_refs across schemas.
+* Added ability to autoadd properties structured_data.StructuredDataSet;
+  to automatically pass in submission_centers on submission, and
+  not require that the user explicitly set this in the spreadsheet.
 * Changes to structured_data to respect uniqueItems for arrays.
 * Handle no schemas better in structured_data.
 * Added portal_utils.Portal.ping().

dcicutils/portal_utils.py

@@ -23,15 +23,14 @@ class Portal:
     2. From a key dictionary, containing "key" and "secret" property values.
     3. From a key tuple, containing (in order) a key and secret values.
     4. From a keys file assumed to reside in ~/.{app}-keys.json where the given "app" value is either "smaht", "cgap",
-       or "fourfront"; and where this file is assumed to contain a dictionary with a key equal to the given "env"
-       value (e.g. smaht-localhost) and with a dictionary value containing "key" and "secret" property values; if
-       an "app" value is not specified but the given "env" value begins with one of the app values then that value
-       will be used, i.e. e.g. if env is "smaht-localhost" and app is unspecified than it is assumed to be "smaht".
+       or "fourfront"; where is assumed to contain a dictionary with a key for the given "env" value, e.g. smaht-local;
+       and with a dictionary value containing "key" and "secret" property values, and an optional "server" property;
+       if an "app" value is not specified but the given "env" value begins with one of the app values then that value
+       will be used, i.e. e.g. if "env" is "smaht-local" and app is unspecified than it is assumed to be "smaht".
     5. From a keys file as described above (#4) but rather than be identified by the given "env" value it
-       is looked up by the given "server" name and the "server" key dictionary value in the key file.
+       is looked up via the given "server" name and the "server" key dictionary value in the key file.
     6. From a given "vapp" value (which is assumed to be a TestApp or VirtualApp).
-    7. From another Portal object.
-    8. From a a pyramid Router object.
+    7. From another Portal object; or from a a pyramid Router object.
     """
     def __init__(self,
                  arg: Optional[Union[VirtualApp, TestApp, Router, Portal, dict, tuple, str]] = None,

dcicutils/structured_data.py

@@ -42,8 +42,8 @@
 class StructuredDataSet:
 
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-                 schemas: Optional[List[dict]] = None, data: Optional[List[dict]] = None,
-                 order: Optional[List[str]] = None, prune: bool = True) -> None:
+                 schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
+                 data: Optional[List[dict]] = None, order: Optional[List[str]] = None, prune: bool = True) -> None:
         self.data = {} if not data else data  # If portal is None then no schemas nor refs.
         self._portal = Portal(portal, data=self.data, schemas=schemas) if portal else None
         self._order = order
@@ -52,13 +52,14 @@ def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp
         self._errors = {}
         self._resolved_refs = set()
         self._validated = False
+        self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._load_file(file) if file else None
 
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
-             schemas: Optional[List[dict]] = None,
+             schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
              order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
-        return StructuredDataSet(file=file, portal=portal, schemas=schemas, order=order, prune=prune)
+        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
         if self._validated and not force:
@@ -163,6 +164,8 @@ def _load_reader(self, reader: RowReader, type_name: str) -> None:
             structured_row = structured_row_template.create_row()
             for column_name, value in row.items():
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
+                if self._autoadd_properties:
+                    self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
         self._note_warning(reader.warnings, "reader")
         if schema:
@@ -177,6 +180,11 @@ def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
         else:
             self.data[type_name] = [data] if isinstance(data, dict) else data
 
+    def _add_properties(self, structured_row: dict, properties: dict, schema: Optional[dict] = None) -> None:
+        for name in properties:
+            if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
+                structured_row[name] = properties[name]
+
     def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
         self._note_issue(self._warnings, item, group)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.5.0.1b3"  # TODO: To become 8.6.0
+version = "8.5.0.1b4"  # TODO: To become 8.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"