Merge pull request #34 from RalphTro/embedded_user_extensions

Embedded user extensions

.gitignore

@@ -105,4 +105,5 @@ venv.bak/
 
 # tmp files
 \#*#
-*~
+*~
+.vscode/*

README.md

@@ -141,10 +141,14 @@ Example 1:
 
 ![Example 1 for EPCIS event pre-hash computation](docs/hashingAlgorithmLogicIllustration_example1.jpg)
 
+Run `epcis_event_hash_generator/main.py tests/examples/ReferenceEventHashAlgorithm.xml -pj "\n"` to get a similar output of the pre-hash string and `epcis_event_hash_generator/main.py tests/examples/ReferenceEventHashAlgorithm.xml` to verify the hash.
+
 Example 2:
 
 ![Example 2 for EPCIS event pre-hash computation ](docs/hashingAlgorithmLogicIllustration_example2.jpg)
 
+Run `epcis_event_hash_generator/main.py tests/examples/ReferenceEventHashAlgorithm2.xml -pj "\n"` to get a similar output of the pre-hash string and `epcis_event_hash_generator/main.py tests/examples/ReferenceEventHashAlgorithm2.xml` to verify the hash.
+
 Example 3:
 
 ![Example 3 for EPCIS event pre-hash computation ](docs/hashingAlgorithmLogicIllustration_example3.jpg)

epcis_event_hash_generator/__init__.py

@@ -1,5 +1,13 @@
 # -*- coding: utf-8 -*-
 
+JOIN_BY = ""
+"""
+Join the substrings to the pre hash string using this deliminator.
+By the specification in https://github.com/RalphTro/epcis-event-hash-generator this is to be the empty string,
+but using e.g. newline might be helpful for debugging.
+When using the command line utility, this can be changed via the -j flag.
+"""
+
 PROP_ORDER = [
     ('eventTime', None),
     ('eventTimeZoneOffset', None),

epcis_event_hash_generator/context.py

@@ -1,6 +1,5 @@
+import epcis_event_hash_generator
 import os
 import sys
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
-import epcis_event_hash_generator

epcis_event_hash_generator/hash_generator.py

@@ -20,6 +20,7 @@
 import datetime
 import hashlib
 import logging
+import traceback
 
 import dateutil.parser
 
@@ -33,6 +34,9 @@
 from epcis_event_hash_generator.json_to_py import event_list_from_epcis_document_json as read_json
 from epcis_event_hash_generator.json_to_py import event_list_from_epcis_document_json_str as read_json_str
 from epcis_event_hash_generator import PROP_ORDER
+from epcis_event_hash_generator import JOIN_BY as DEFAULT_JOIN_BY
+
+JOIN_BY = DEFAULT_JOIN_BY
 
 
 def fix_time_stamp_format(timestamp):
@@ -55,39 +59,68 @@ def fix_time_stamp_format(timestamp):
     return fixed
 
 
-def recurse_through_children_in_order(root, child_order):
-    """child_order is expected to be a property order, see PROP_ORDER. Loop over child order, look for a child of
-    root with matching name and add its text (if any). Recurse through the grand children applying the sub order.
+def child_to_pre_hash_string(child, sub_child_order):
+    text = ""
+    grand_child_text = ""
+    if sub_child_order:
+        grand_child_text = recurse_through_children_in_order(child[2], sub_child_order)
+    if child[1]:
+        text = child[1].strip()
+        if child[0].lower().find("time") > 0 and child[0].lower().find("offset") < 0:
+            text = fix_time_stamp_format(text)
+        else:
+            text = format_if_numeric(text)
+
+        if text:
+            text = "=" + text
+            logging.debug("Adding text '%s'", text)
+
+    if text or grand_child_text:
+        return child[0] + text + grand_child_text
+
+    return ""
+
+
+def recurse_through_children_in_order(child_list, child_order):
+    """
+    Loop over child order, look for a child of root with matching key and build the pre-hash string (mostly key=value)
+    Recurse through the grand children applying the sub order.
+    All elements added to the returned pre hash string are removed from the tree below the root.
+    After the recursion completes, only elements NOT added to the pre-hash string are left in the tree.
+
+    `child_list`    is to be a list of simple python object, i.e. triples of two strings (key/value) and a list of
+                    simple python objects (grand children).
+    `child_order`   is expected to be a property order, see PROP_ORDER.
+
     """
-    texts = ""
+    pre_hash = ""
+    logging.debug("Calculating pre hash for child list %s \nWith order %s", child_list, child_order)
     for (child_name, sub_child_order) in child_order:
+        children = [x for x in child_list if x[0] == child_name]  # elements with the same name
         list_of_values = []
-        prefix = ""
-        for child in [x for x in root if x[0] == child_name]:
-            if sub_child_order:
-                list_of_values.append(recurse_through_children_in_order(child[2], sub_child_order))
-                prefix = child_name
-            if child[1]:
-                text = child[1].strip()
-                if child_name.lower().find("time") > 0 and child_name.lower().find("offset") < 0:
-                    text = fix_time_stamp_format(text)
-                else:
-                    text = format_if_numeric(text)
-
-                logging.debug("Adding text '%s'", text)
-                list_of_values.append(
-                    child_name + "=" + text)
-
-        # sort list of values to fix !10
+
+        for child in children:
+            child_pre_hash = child_to_pre_hash_string(child, sub_child_order)
+            if child_pre_hash:
+                list_of_values.append(child_pre_hash)
+            else:
+                logging.debug("Empty element ignored: %s", child)
+
+            if len(child[2]) == 0:
+                logging.debug("Finished processing %s", child)
+                child_list.remove(child)
+
+        # sort list of values to fix #10
         list_of_values.sort()
-        if len(list_of_values) > 1:
-            logging.debug("sorted: %s", list_of_values)
 
         if "".join(list_of_values):  # fixes #16
-            texts += prefix + "".join(list_of_values)
-        elif prefix:
-            logging.debug("Skipping empty element: %s", prefix)
-    return texts
+            if pre_hash:
+                list_of_values.insert(0, pre_hash)  # yields correct Joining behavior
+            pre_hash = JOIN_BY.join(list_of_values)
+
+    logging.debug("child list pre hash is %s", pre_hash)
+
+    return pre_hash
 
 
 def format_if_numeric(text):
@@ -102,38 +135,32 @@ def format_if_numeric(text):
     return text
 
 
-def generic_element_to_prehash_string(root):
+def generic_child_list_to_prehash_string(children):
     list_of_values = []
 
-    logging.debug("Parsing remaining elements: %s", root)
-    if isinstance(root, str) and root:
-        list_of_values.append("=" + root.strip())
-    else:
-        for child in root:
-            list_of_values.append(child[0] + generic_element_to_prehash_string(
-                child[1]) + generic_element_to_prehash_string(child[2]))
+    logging.debug("Parsing remaining elements in: %s", children)
+
+    for child in children:
+        text = child[1].strip()
+        if text:
+            text = "=" + text
+        list_of_values.append(child[0] + text + generic_child_list_to_prehash_string(child[2]))
 
     list_of_values.sort()
-    return "".join(list_of_values)
+    return JOIN_BY.join(list_of_values)
 
 
-def gather_elements_not_in_order(root, child_order):
+def gather_elements_not_in_order(children, child_order):
     """
     Collects vendor extensions not covered by the defined child order. Consumes the root.
     """
 
     # remove recordTime, if any
-    child_order_or_record_time = child_order + [("recordTime", None)]
-
-    for (child_name, _) in child_order_or_record_time:
-        covered_children = [x for x in root if x[0] == child_name]
-        logging.debug("Children '%s' covered by ordering: %s", child_name, covered_children)
-        for child in covered_children:
-            root.remove(child)
-
-    logging.debug("Parsing remaining elements in: %s", root)
-    if root:
-        return generic_element_to_prehash_string(root)
+    for child in children:
+        if child[0] == "recordTime":
+            children.remove(child)
+    if children:
+        return generic_child_list_to_prehash_string(children)
 
     return ""
 
@@ -179,12 +206,13 @@ def compute_prehash_from_events(events):
     for event in events[2]:
         logging.debug("prehashing event:\n%s", event)
         try:
-            prehash_string_list.append("eventType=" + event[0] +
-                                       recurse_through_children_in_order(event[2], PROP_ORDER)
+            prehash_string_list.append("eventType=" + event[0] + JOIN_BY
+                                       + recurse_through_children_in_order(event[2], PROP_ORDER) + JOIN_BY
                                        + gather_elements_not_in_order(event[2], PROP_ORDER)
                                        )
         except Exception as ex:
             logging.error("could not parse event:\n%s\n\nerror: %s", event, ex)
+            logging.debug("".join(traceback.format_tb(ex.__traceback__)))
             pass
 
     # To see/check concatenated value string before hash algorithm is performed:
@@ -202,12 +230,16 @@ def epcis_hash_from_xml(xmlStr, hashalg="sha256"):
     return calculate_hash(prehash_string_list, hashalg)
 
 
-def epcis_hash(path, hashalg="sha256"):
+def epcis_hash(path, hashalg="sha256", join_by=DEFAULT_JOIN_BY):
     """Read all EPCIS Events from the EPCIS XML document at path.
     Compute a normalized form (pre-hash string) for each event and
     return an array of the event hashes computed from the pre-hash by
     hashalg.
     """
+    global JOIN_BY
+    join_by = join_by.replace(r"\n", "\n").replace(r"\t", "\t")
+    logging.debug("Setting JOIN_BY='%s'", join_by)
+    JOIN_BY = join_by
     prehash_string_list = compute_prehash_from_file(path)
 
     return calculate_hash(prehash_string_list, hashalg)

epcis_event_hash_generator/main.py

@@ -65,6 +65,12 @@ def command_line_parsing():
         "--prehash",
         help="If given, also output the prehash string to stdout. Output to a .prehashes file, if combined with -b.",
         action="store_true")
+    parser.add_argument(
+        "-j",
+        "--join",
+        help="String used to join the pre hash string." +
+        " Defaults to empty string as specified. Values like '\\n' might be useful for debugging.",
+        default="")
 
     args = parser.parse_args()
 
@@ -91,9 +97,11 @@ def main():
 
     args = command_line_parsing()
 
+    logging.debug("Running cli tool with arguments %s", args)
+
     for filename in args.file:
         # ACTUAL ALGORITHM CALL:
-        (hashes, prehashes) = hash_generator.epcis_hash(filename, args.algorithm)
+        (hashes, prehashes) = hash_generator.epcis_hash(filename, args.algorithm, args.join)
 
         # Output:
         if args.batch:

tests/examples/ReferenceEventHashAlgorithm2.hashes

@@ -1 +1 @@
-ni:///sha-256;36da11936e0e528740bd1767bccbe568d60758e627f2c8a8c86c8a72af8038c8?ver=CBV2.0
+ni:///sha-256;0974aa383919495ccff808b0a57123143545ea7f5f9e9177b0c547db1a1e6d70?ver=CBV2.0

tests/examples/ReferenceEventHashAlgorithm2.prehashes

@@ -1 +1 @@
-eventType=ObjectEventeventTime=2020-04-01T14:00:00.000ZeventTimeZoneOffset=+01:00epcListepc=urn:epc:id:sgtin:4012345.011111.9876action=OBSERVEbizStep=urn:epcglobal:cbv:bizstep:inspectingreadPointid=urn:epc:id:sgln:4012345.00005.0sensorElementListsensorElementsensorMetaDatadeviceID=urn:epc:id:giai:4000001.111deviceMetaData=https://id.gs1.org/giai/4000001111sensorReporttype=gs1:Humidityvalue=12.1uom=A93type=gs1:Molar_concentrationchemicalSubstance=urn:epcglobal:cbv:inchikey:CZMRCDWAGMRECN-UGDNZRGBSA-Nvalue=0.18uom=C35type=gs1:Molar_concentrationmicroorganism=https://www.ncbi.nlm.nih.gov/taxonomy/1126011value=0.05uom=C35type=gs1:Temperaturevalue=26sDev=0.1uom=CEL
+eventType=ObjectEventeventTime=2020-04-01T14:00:00.000ZeventTimeZoneOffset=+01:00epcListepc=urn:epc:id:sgtin:4012345.011111.9876action=OBSERVEbizStep=urn:epcglobal:cbv:bizstep:inspectingreadPointid=urn:epc:id:sgln:4012345.00005.0sensorElementListsensorElementsensorMetaDatadeviceID=urn:epc:id:giai:4000001.111deviceMetaData=https://id.gs1.org/giai/4000001111sensorReporttype=gs1:Humidityvalue=12.1uom=A93sensorReporttype=gs1:Molar_concentrationchemicalSubstance=urn:epcglobal:cbv:inchikey:CZMRCDWAGMRECN-UGDNZRGBSA-Nvalue=0.18uom=C35sensorReporttype=gs1:Molar_concentrationmicroorganism=https://www.ncbi.nlm.nih.gov/taxonomy/1126011value=0.05uom=C35sensorReporttype=gs1:Temperaturevalue=26sDev=0.1uom=CEL

tests/examples/SensorDataExamples.hashes

@@ -1,12 +1,12 @@
-ni:///sha-256;665a51da93e2a52ced894282f600fce1e67162d683a9e997eaa0657e53da4909?ver=CBV2.0
-ni:///sha-256;e46f44d8d6c8a2f156d64672afc704d4458133a65a1f01c8df61dbce6e2ad8be?ver=CBV2.0
-ni:///sha-256;6f3644012982d8b9dfb27aa33f4f3e6720a98df087e044762c19a95340d6babd?ver=CBV2.0
-ni:///sha-256;7fef66f4027ec7ed7a156f1ccd5bc4a9cdb686c24e128d0b43df1d70943c27ef?ver=CBV2.0
-ni:///sha-256;33bdeed428bb37e6404decbcd2e395fd3651ae6a3f19cd4c39ecdc704935c25e?ver=CBV2.0
-ni:///sha-256;10904fdf6331a91a9f3073fdedab3dc8484efb08a7cf896c46bcdf8c6b0d758d?ver=CBV2.0
-ni:///sha-256;b415e8c0ddf693e8e8b485b5b70df813c7b9ba71b5ca2918aeca3e18b0b9f563?ver=CBV2.0
-ni:///sha-256;41361e3404b1da90c09dabdb5aa3b42bd61fcf25395a2d441cd52c9eacfe8bc2?ver=CBV2.0
-ni:///sha-256;01eab3209f0b170b10ceca4f9d7b15a5e0dce0dd71f67207eb18af14df89273a?ver=CBV2.0
-ni:///sha-256;1d1818e087f6acb597e221d51cf1e57f7aa5977e16a0d1deeb8748b22637e298?ver=CBV2.0
-ni:///sha-256;0df156086a615ffa6eb7a73ed2b5400ada7ea545d38c7dd79c8e1dd3b5fac89e?ver=CBV2.0
-ni:///sha-256;cbcb8a828594f56a8c352fb31ed0c7ff58427ecaf4e8795c2533871c28facff6?ver=CBV2.0
+ni:///sha-256;8bd609d4bbedfd7a2af38c6fbc222d66a83988859eee372ac50f3079795fe4b8?ver=CBV2.0
+ni:///sha-256;47818322cd49362b5ef8f0747f726ee21d02b2b381bf28f15522e09884a29e35?ver=CBV2.0
+ni:///sha-256;d6acbd943a9dd30b3f10b5e45fa64a0b21b1c91954cf24ab45d6d914e9e40c75?ver=CBV2.0
+ni:///sha-256;91c7021c96b5a5df2a174e487640d3e0b5ab1f3cf2ed2f518b2f662e1fd2c4bf?ver=CBV2.0
+ni:///sha-256;28b6d85591e631e9d7bd64e3dbe40d3e03384b29844e1648d130f1b390f08bbd?ver=CBV2.0
+ni:///sha-256;6a86564b861d2ce1d2d7273bec9363a311a7f727edaa828003965f10d384200b?ver=CBV2.0
+ni:///sha-256;63e2d119c69c91c653027895db05492bac2211dacee0d7f9239146fc566d2192?ver=CBV2.0
+ni:///sha-256;a84a793d7a9f823332f41118eb973caf86e673048e532a2b850616547bbbb4cf?ver=CBV2.0
+ni:///sha-256;546f0b817f058290745fbab2b28d6ff142b06207df8bfcc13b0bf72c609e84b1?ver=CBV2.0
+ni:///sha-256;1f560c5829e959cab2473ddc734ea90a088eb5202e3c611a3108a04897932f0a?ver=CBV2.0
+ni:///sha-256;8702ac80df8ac48d19cc0c8d19399249fc7037f8b752acb97baf7148960d2a17?ver=CBV2.0
+ni:///sha-256;6d13f864b06d22b4cd0a97251a21ae32d309ccb63457652ce946f5436a25a61b?ver=CBV2.0