Merge pull request #114 from RalphTro/support-for-ignore-fields

support for ignoring industry/repository specific fields

epcis_event_hash_generator/json_to_py.py

@@ -93,10 +93,12 @@ def _collect_namespaces_from_jsonld_context(context):
                         _namespaces[key] = "{" + c[key] + "}"
 
 
-def _json_to_py(json_obj):
+def _json_to_py(json_obj, fields_to_ignore=None):
     """
     Recursively convert a string/list/dict to a simple python object
     """
+    if fields_to_ignore is None:
+        fields_to_ignore = []
     global _namespaces
 
     py_obj = ("", "", [])
@@ -111,7 +113,7 @@ def _json_to_py(json_obj):
         if "#text" in json_obj:
             py_obj = (py_obj[0], json_obj["#text"], py_obj[2])
 
-        to_be_ignored = ["#text", "rdfs:comment", "comment"]
+        to_be_ignored = ["#text", "rdfs:comment", "comment"] + fields_to_ignore
         for (key, val) in [x for x in json_obj.items() if x[0] not in to_be_ignored]:
             if key.startswith("@xmlns"):
                 _namespaces[key[7:]] = "{" + val + "}"
@@ -244,18 +246,28 @@ def event_list_from_epcis_document_json(json_obj):
     if not json_obj.get("@context") is None:
         _collect_namespaces_from_jsonld_context(json_obj["@context"])
 
+    internal_domain = '{https://repository-x.example.com/}'
+    ignore_field_key = ':ignoreFields'
+    for key, value in _namespaces.items():
+        if value == internal_domain:
+            ignore_field_key = key+ignore_field_key
+            break
+
     if "eventList" in json_obj["epcisBody"]:
         event_list = json_obj["epcisBody"]["eventList"]
+        fields_to_ignore = json_obj.get(ignore_field_key)
     elif "queryResults" in json_obj["epcisBody"]:
         event_list = json_obj["epcisBody"]["queryResults"]["resultsBody"]["eventList"]
+        fields_to_ignore = json_obj["epcisBody"]["queryResults"].get(ignore_field_key)
     else:
         # epcisBody may contain single event
         event_list = [json_obj["epcisBody"]["event"]]
+        fields_to_ignore = json_obj.get(ignore_field_key)
 
     events = []
 
     # Correct JSON/XML data model mismatch
     for event in event_list:
-        events.append(json_xml_model_mismatch_correction.deep_structure_correction(_json_to_py(event)))
+        events.append(json_xml_model_mismatch_correction.deep_structure_correction(_json_to_py(event, fields_to_ignore)))
 
     return ("EventList", "", events)

epcis_event_hash_generator/xml_to_py.py

@@ -65,6 +65,7 @@
 
 import logging
 import xml.etree.ElementTree as ElementTree
+from lxml import etree
 from typing import Tuple
 
 _expansions = {"gs1:": "https://gs1.org/voc/", "cbv:": "https://ref.gs1.org/cbv/"}
@@ -168,13 +169,46 @@ def _xml_to_py(root, sort=True):
     return obj
 
 
+def remove_xml_declaration(xml_string):
+    """
+    Removes the <?xml> tag from the beginning of an XML string if present.
+    """
+    if xml_string.startswith("<?xml"):
+        # Find the end of the processing instruction
+        end_pos = xml_string.find("?>") + 2  # Include the ?> characters
+        return xml_string[end_pos:]
+    else:
+        return xml_string
+
+
+def get_ignore_field_prefix_ns(xmlStr: str):
+    """
+    if presents, gets all fields to be ignored from events of EPCIS document.
+    """
+    xml_str_without_decl = remove_xml_declaration(xmlStr)
+
+    all_namespaces = etree.fromstring(xml_str_without_decl).nsmap
+
+    ignore_field_ns_prefix = None
+    for key, value in all_namespaces.items():
+        if value == 'https://repository-x.example.com/':
+            return key
+
+    return ignore_field_ns_prefix
+
+
 def event_list_from_epcis_document_str(xmlStr: str) -> Tuple[str, str, list]:
     """
     Read EPCIS XML document and generate the event List in the form of a simple python object
     """
     try:
         data = _remove_extension_tags(xmlStr)
 
+        ignore_field_ns_prefix = get_ignore_field_prefix_ns(xmlStr)
+
+        if ignore_field_ns_prefix is not None:
+            data = data.replace(ignore_field_ns_prefix + ':', '')
+
         root = ElementTree.fromstring(data)
 
         eventList = root.find("*EventList")
@@ -184,6 +218,14 @@ def event_list_from_epcis_document_str(xmlStr: str) -> Tuple[str, str, list]:
 
         if not eventList:
             raise ValueError("No EventList found")
+
+        # remove all fields to be ignored
+        for field_to_remove in root.findall("ignoreFields/*"):
+            for event in eventList:
+                el = event.find(field_to_remove.tag)
+                if el is not None:
+                    event.remove(el)
+
     except (ValueError, OSError) as ex:
         logging.error(ex)
         logging.error("Input string does not contain a valid EPCIS XML document with EventList.")

tests/examples/epcisDocHavingEventWithIgnoreFields.hashes

@@ -0,0 +1,2 @@
+ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0
+ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0

tests/examples/epcisDocHavingEventWithIgnoreFields.jsonld

@@ -0,0 +1,55 @@
+{
+  "@context": [
+    "https://ref.gs1.org/standards/epcis/2.0.0/epcis-context.jsonld",
+    {
+      "example": "https://ns.example.com/epcis/",
+      "repository-x": "https://repository-x.example.com/"
+    },
+    {
+      "rdfs": "http://www.w3.org/2000/01/rdf-schema#"
+    }
+  ],
+  "id": "https://id.example.org/document1",
+  "type": "EPCISDocument",
+  "schemaVersion": "2.0",
+  "creationDate": "2005-07-11T11:30:47.0Z",
+  "repository-x:ignoreFields": ["repository-x:hash", "example:captureID"],
+  "epcisBody": {
+    "eventList": [
+        {
+            "type": "ObjectEvent",
+            "eventTime": "2019-10-21T14:45:00Z",
+            "recordTime": "2023-06-02T07:03:16.073Z",
+            "eventTimeZoneOffset": "+01:00",
+            "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+            "epcList": [
+                "https://id.gs1.org/00/040123451111111127"
+            ],
+            "action": "OBSERVE",
+            "bizStep": "inspecting",
+            "disposition": "in_progress",
+            "readPoint": {
+                "id": "https://id.gs1.org/414/4012345000245/254/1"
+            },
+            "repository-x:hash": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+            "example:captureID": "cc90bbe4-7f3a-4abd-b150-d4371c72a64f"
+        },
+        {
+            "type": "ObjectEvent",
+            "eventTime": "2019-10-21T14:45:00Z",
+            "recordTime": "2023-06-02T07:03:16.073Z",
+            "eventTimeZoneOffset": "+01:00",
+            "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+            "epcList": [
+                "https://id.gs1.org/00/040123451111111127"
+            ],
+            "action": "OBSERVE",
+            "bizStep": "inspecting",
+            "disposition": "in_progress",
+            "readPoint": {
+                "id": "https://id.gs1.org/414/4012345000245/254/1"
+            }
+        }
+    ]
+  }
+}

tests/examples/epcisDocHavingEventWithIgnoreFields.prehashes

@@ -0,0 +1,2 @@
+eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1
+eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1

tests/examples/epcisQueryDocHavingEventWithIgnoreFields.hashes

@@ -0,0 +1,2 @@
+ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0
+ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0

tests/examples/epcisQueryDocHavingEventWithIgnoreFields.jsonld

@@ -0,0 +1,57 @@
+{
+    "@context": [
+      "https://ref.gs1.org/standards/epcis/2.0.0/epcis-context.jsonld",
+      {
+        "example": "https://ns.example.com/epcis/",
+        "repository-x": "https://repository-x.example.com/"
+      }
+    ],
+    "type": "EPCISQueryDocument",
+    "schemaVersion": "2.0",
+    "creationDate": "2005-07-11T11:30:47.0Z",
+    "epcisBody": {
+      "queryResults": {
+        "subscriptionID": "32d2aec1-a6d2-46d9-900a-24124288cce1",
+        "queryName": "SimpleEventQuery",
+        "repository-x:ignoreFields": ["repository-x:hash", "example:captureID"],
+        "resultsBody": {
+          "eventList": [
+            {
+                "type": "ObjectEvent",
+                "eventTime": "2019-10-21T14:45:00Z",
+                "recordTime": "2023-06-02T07:03:16.073Z",
+                "eventTimeZoneOffset": "+01:00",
+                "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+                "epcList": [
+                    "https://id.gs1.org/00/040123451111111127"
+                ],
+                "action": "OBSERVE",
+                "bizStep": "inspecting",
+                "disposition": "in_progress",
+                "readPoint": {
+                    "id": "https://id.gs1.org/414/4012345000245/254/1"
+                },
+                "repository-x:hash": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+                "example:captureID": "cc90bbe4-7f3a-4abd-b150-d4371c72a64f"
+            },
+            {
+                "type": "ObjectEvent",
+                "eventTime": "2019-10-21T14:45:00Z",
+                "recordTime": "2023-06-02T07:03:16.073Z",
+                "eventTimeZoneOffset": "+01:00",
+                "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0",
+                "epcList": [
+                    "https://id.gs1.org/00/040123451111111127"
+                ],
+                "action": "OBSERVE",
+                "bizStep": "inspecting",
+                "disposition": "in_progress",
+                "readPoint": {
+                    "id": "https://id.gs1.org/414/4012345000245/254/1"
+                }
+            }
+          ]
+        }
+      }
+    }
+  }

tests/examples/epcisQueryDocHavingEventWithIgnoreFields.prehashes

@@ -0,0 +1,2 @@
+eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1
+eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1

tests/examples/epcisXmlDocHavingEventWithIgnoreFields.hashes

@@ -0,0 +1,2 @@
+ni:///sha-256;2e3080ec5bfad98586d73f1943fae80e201613c1b2c0f084b3b8cd5976aaea5c?ver=CBV2.0
+ni:///sha-256;bfcf5b87dd396d4f7b266e33d4a969dc5250344ed0e47e85ce8e26d1adadad11?ver=CBV2.0

tests/examples/epcisXmlDocHavingEventWithIgnoreFields.prehashes

@@ -0,0 +1,2 @@
+eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField4=ABCD
+eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField3=PQRS

tests/examples/epcisXmlDocHavingEventWithIgnoreFields.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<epcis:EPCISDocument xmlns:epcis="urn:epcglobal:epcis:xsd:2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                     xmlns:cbv="https://ref.gs1.org/cbv/"
+                     schemaVersion="2.0"
+                     creationDate="2024-06-08T12:00:00.000+01:00"
+                     xsi:schemaLocation="urn:epcglobal:epcis:xsd:2 EPCglobal-epcis-2_0.xsd"
+                     xmlns:repository-x="https://repository-x.example.com/"
+                     xmlns:example="https://ns.example.com/epcis/">
+    <repository-x:ignoreFields>
+        <example:testField1/>
+        <example:testField2/>
+    </repository-x:ignoreFields>
+    <EPCISBody>
+        <EventList>
+            <ObjectEvent>
+                <eventTime>2024-06-08T10:58:56.591+02:00</eventTime>
+                <recordTime>2024-06-08T15:38:32.052+02:00</recordTime>
+                <eventTimeZoneOffset>+02:00</eventTimeZoneOffset>
+                <epcList/>
+                <action>ADD</action>
+                <bizStep>urn:epcglobal:cbv:bizstep:commissioning</bizStep>
+                <readPoint>
+                    <id>urn:epc:id:sgln:4054738.00005.0</id>
+                </readPoint>
+                <extension>
+                    <quantityList>
+                        <quantityElement>
+                            <epcClass>urn:epc:class:lgtin:4054739.099961.2015-03-30a</epcClass>
+                            <quantity>150</quantity>
+                            <uom>KGM</uom>
+                        </quantityElement>
+                    </quantityList>
+                </extension>
+                <example:testField1>1234</example:testField1>
+                <example:testField4>ABCD</example:testField4>
+            </ObjectEvent>
+            <ObjectEvent>
+                <eventTime>2024-06-08T10:58:56.591+02:00</eventTime>
+                <recordTime>2024-06-08T15:38:32.052+02:00</recordTime>
+                <eventTimeZoneOffset>+02:00</eventTimeZoneOffset>
+                <epcList/>
+                <action>ADD</action>
+                <bizStep>urn:epcglobal:cbv:bizstep:commissioning</bizStep>
+                <readPoint>
+                    <id>urn:epc:id:sgln:4054738.00005.0</id>
+                </readPoint>
+                <extension>
+                    <quantityList>
+                        <quantityElement>
+                            <epcClass>urn:epc:class:lgtin:4054739.099961.2015-03-30a</epcClass>
+                            <quantity>150</quantity>
+                            <uom>KGM</uom>
+                        </quantityElement>
+                    </quantityList>
+                </extension>
+                <example:testField2>1234</example:testField2>
+                <example:testField3>PQRS</example:testField3>
+            </ObjectEvent>
+        </EventList>
+    </EPCISBody>
+</epcis:EPCISDocument>

tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.hashes

@@ -0,0 +1,2 @@
+ni:///sha-256;573a43267f95d2ddbe7e7487ad3ffb89e78ee8c72a296b868d9e5baeb8baef34?ver=CBV2.0
+ni:///sha-256;1e6c7bbfc2d3f744e99af09ad8216505b72f0ced383e391f92ab92674dc906c4?ver=CBV2.0

tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.prehashes

@@ -0,0 +1,3 @@
+eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField4=ABC
+eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField3=XYZ
+

tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<epcisq:EPCISQueryDocument xmlns:epcisq="urn:epcglobal:epcis-query:xsd:2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xmlns:cbvmda="urn:epcglobal:cbv:mda"
+                           xmlns:sbdh="http://www.unece.org/cefact/namespaces/StandardBusinessDocumentHeader"
+                           schemaVersion="2.0"
+                           creationDate="2024-06-08T11:30:47.0Z"
+                           xmlns:repository-x="https://repository-x.example.com/"
+                           xmlns:example="https://ns.example.com/epcis/">
+    <repository-x:ignoreFields>
+        <example:testField1/>
+        <example:testField2/>
+    </repository-x:ignoreFields>
+    <queryName>SimpleEventQuery</queryName>
+    <resultsBody>
+        <EventList>
+            <ObjectEvent>
+                <eventTime>2024-06-08T10:58:56.591+02:00</eventTime>
+                <recordTime>2024-06-08T15:38:32.052+02:00</recordTime>
+                <eventTimeZoneOffset>+02:00</eventTimeZoneOffset>
+                <epcList/>
+                <action>ADD</action>
+                <bizStep>urn:epcglobal:cbv:bizstep:commissioning</bizStep>
+                <readPoint>
+                    <id>urn:epc:id:sgln:4054738.00005.0</id>
+                </readPoint>
+                <extension>
+                    <quantityList>
+                        <quantityElement>
+                            <epcClass>urn:epc:class:lgtin:4054739.099961.2015-03-30a</epcClass>
+                            <quantity>150</quantity>
+                            <uom>KGM</uom>
+                        </quantityElement>
+                    </quantityList>
+                </extension>
+                <example:testField1>123</example:testField1>
+                <example:testField4>ABC</example:testField4>
+            </ObjectEvent>
+            <ObjectEvent>
+                <eventTime>2024-06-08T10:58:56.591+02:00</eventTime>
+                <recordTime>2024-06-08T15:38:32.052+02:00</recordTime>
+                <eventTimeZoneOffset>+02:00</eventTimeZoneOffset>
+                <epcList/>
+                <action>ADD</action>
+                <bizStep>urn:epcglobal:cbv:bizstep:commissioning</bizStep>
+                <readPoint>
+                    <id>urn:epc:id:sgln:4054738.00005.0</id>
+                </readPoint>
+                <extension>
+                    <quantityList>
+                        <quantityElement>
+                            <epcClass>urn:epc:class:lgtin:4054739.099961.2015-03-30a</epcClass>
+                            <quantity>150</quantity>
+                            <uom>KGM</uom>
+                        </quantityElement>
+                    </quantityList>
+                </extension>
+                <example:testField2>123</example:testField2>
+                <example:testField3>XYZ</example:testField3>
+            </ObjectEvent>
+        </EventList>
+    </resultsBody>
+</epcisq:EPCISQueryDocument>