diff --git a/epcis_event_hash_generator/json_to_py.py b/epcis_event_hash_generator/json_to_py.py index ca0e3ce..4efac52 100644 --- a/epcis_event_hash_generator/json_to_py.py +++ b/epcis_event_hash_generator/json_to_py.py @@ -93,10 +93,12 @@ def _collect_namespaces_from_jsonld_context(context): _namespaces[key] = "{" + c[key] + "}" -def _json_to_py(json_obj): +def _json_to_py(json_obj, fields_to_ignore=None): """ Recursively convert a string/list/dict to a simple python object """ + if fields_to_ignore is None: + fields_to_ignore = [] global _namespaces py_obj = ("", "", []) @@ -111,7 +113,7 @@ def _json_to_py(json_obj): if "#text" in json_obj: py_obj = (py_obj[0], json_obj["#text"], py_obj[2]) - to_be_ignored = ["#text", "rdfs:comment", "comment"] + to_be_ignored = ["#text", "rdfs:comment", "comment"] + fields_to_ignore for (key, val) in [x for x in json_obj.items() if x[0] not in to_be_ignored]: if key.startswith("@xmlns"): _namespaces[key[7:]] = "{" + val + "}" @@ -244,18 +246,28 @@ def event_list_from_epcis_document_json(json_obj): if not json_obj.get("@context") is None: _collect_namespaces_from_jsonld_context(json_obj["@context"]) + internal_domain = '{https://repository-x.example.com/}' + ignore_field_key = ':ignoreFields' + for key, value in _namespaces.items(): + if value == internal_domain: + ignore_field_key = key+ignore_field_key + break + if "eventList" in json_obj["epcisBody"]: event_list = json_obj["epcisBody"]["eventList"] + fields_to_ignore = json_obj.get(ignore_field_key) elif "queryResults" in json_obj["epcisBody"]: event_list = json_obj["epcisBody"]["queryResults"]["resultsBody"]["eventList"] + fields_to_ignore = json_obj["epcisBody"]["queryResults"].get(ignore_field_key) else: # epcisBody may contain single event event_list = [json_obj["epcisBody"]["event"]] + fields_to_ignore = json_obj.get(ignore_field_key) events = [] # Correct JSON/XML data model mismatch for event in event_list: - events.append(json_xml_model_mismatch_correction.deep_structure_correction(_json_to_py(event))) + events.append(json_xml_model_mismatch_correction.deep_structure_correction(_json_to_py(event, fields_to_ignore))) return ("EventList", "", events) diff --git a/epcis_event_hash_generator/xml_to_py.py b/epcis_event_hash_generator/xml_to_py.py index 3a9bc0c..d965f9d 100644 --- a/epcis_event_hash_generator/xml_to_py.py +++ b/epcis_event_hash_generator/xml_to_py.py @@ -65,6 +65,7 @@ import logging import xml.etree.ElementTree as ElementTree +from lxml import etree from typing import Tuple _expansions = {"gs1:": "https://gs1.org/voc/", "cbv:": "https://ref.gs1.org/cbv/"} @@ -168,6 +169,34 @@ def _xml_to_py(root, sort=True): return obj +def remove_xml_declaration(xml_string): + """ + Removes the tag from the beginning of an XML string if present. + """ + if xml_string.startswith("") + 2 # Include the ?> characters + return xml_string[end_pos:] + else: + return xml_string + + +def get_ignore_field_prefix_ns(xmlStr: str): + """ + if presents, gets all fields to be ignored from events of EPCIS document. + """ + xml_str_without_decl = remove_xml_declaration(xmlStr) + + all_namespaces = etree.fromstring(xml_str_without_decl).nsmap + + ignore_field_ns_prefix = None + for key, value in all_namespaces.items(): + if value == 'https://repository-x.example.com/': + return key + + return ignore_field_ns_prefix + + def event_list_from_epcis_document_str(xmlStr: str) -> Tuple[str, str, list]: """ Read EPCIS XML document and generate the event List in the form of a simple python object @@ -175,6 +204,11 @@ def event_list_from_epcis_document_str(xmlStr: str) -> Tuple[str, str, list]: try: data = _remove_extension_tags(xmlStr) + ignore_field_ns_prefix = get_ignore_field_prefix_ns(xmlStr) + + if ignore_field_ns_prefix is not None: + data = data.replace(ignore_field_ns_prefix + ':', '') + root = ElementTree.fromstring(data) eventList = root.find("*EventList") @@ -184,6 +218,14 @@ def event_list_from_epcis_document_str(xmlStr: str) -> Tuple[str, str, list]: if not eventList: raise ValueError("No EventList found") + + # remove all fields to be ignored + for field_to_remove in root.findall("ignoreFields/*"): + for event in eventList: + el = event.find(field_to_remove.tag) + if el is not None: + event.remove(el) + except (ValueError, OSError) as ex: logging.error(ex) logging.error("Input string does not contain a valid EPCIS XML document with EventList.") diff --git a/tests/examples/epcisDocHavingEventWithIgnoreFields.hashes b/tests/examples/epcisDocHavingEventWithIgnoreFields.hashes new file mode 100644 index 0000000..0be1de3 --- /dev/null +++ b/tests/examples/epcisDocHavingEventWithIgnoreFields.hashes @@ -0,0 +1,2 @@ +ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0 +ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0 diff --git a/tests/examples/epcisDocHavingEventWithIgnoreFields.jsonld b/tests/examples/epcisDocHavingEventWithIgnoreFields.jsonld new file mode 100644 index 0000000..e5cfc85 --- /dev/null +++ b/tests/examples/epcisDocHavingEventWithIgnoreFields.jsonld @@ -0,0 +1,55 @@ +{ + "@context": [ + "https://ref.gs1.org/standards/epcis/2.0.0/epcis-context.jsonld", + { + "example": "https://ns.example.com/epcis/", + "repository-x": "https://repository-x.example.com/" + }, + { + "rdfs": "http://www.w3.org/2000/01/rdf-schema#" + } + ], + "id": "https://id.example.org/document1", + "type": "EPCISDocument", + "schemaVersion": "2.0", + "creationDate": "2005-07-11T11:30:47.0Z", + "repository-x:ignoreFields": ["repository-x:hash", "example:captureID"], + "epcisBody": { + "eventList": [ + { + "type": "ObjectEvent", + "eventTime": "2019-10-21T14:45:00Z", + "recordTime": "2023-06-02T07:03:16.073Z", + "eventTimeZoneOffset": "+01:00", + "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "epcList": [ + "https://id.gs1.org/00/040123451111111127" + ], + "action": "OBSERVE", + "bizStep": "inspecting", + "disposition": "in_progress", + "readPoint": { + "id": "https://id.gs1.org/414/4012345000245/254/1" + }, + "repository-x:hash": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "example:captureID": "cc90bbe4-7f3a-4abd-b150-d4371c72a64f" + }, + { + "type": "ObjectEvent", + "eventTime": "2019-10-21T14:45:00Z", + "recordTime": "2023-06-02T07:03:16.073Z", + "eventTimeZoneOffset": "+01:00", + "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "epcList": [ + "https://id.gs1.org/00/040123451111111127" + ], + "action": "OBSERVE", + "bizStep": "inspecting", + "disposition": "in_progress", + "readPoint": { + "id": "https://id.gs1.org/414/4012345000245/254/1" + } + } + ] + } +} diff --git a/tests/examples/epcisDocHavingEventWithIgnoreFields.prehashes b/tests/examples/epcisDocHavingEventWithIgnoreFields.prehashes new file mode 100644 index 0000000..0055eb9 --- /dev/null +++ b/tests/examples/epcisDocHavingEventWithIgnoreFields.prehashes @@ -0,0 +1,2 @@ +eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1 +eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1 diff --git a/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.hashes b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.hashes new file mode 100644 index 0000000..0be1de3 --- /dev/null +++ b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.hashes @@ -0,0 +1,2 @@ +ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0 +ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0 diff --git a/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.jsonld b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.jsonld new file mode 100644 index 0000000..bcbd1b5 --- /dev/null +++ b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.jsonld @@ -0,0 +1,57 @@ +{ + "@context": [ + "https://ref.gs1.org/standards/epcis/2.0.0/epcis-context.jsonld", + { + "example": "https://ns.example.com/epcis/", + "repository-x": "https://repository-x.example.com/" + } + ], + "type": "EPCISQueryDocument", + "schemaVersion": "2.0", + "creationDate": "2005-07-11T11:30:47.0Z", + "epcisBody": { + "queryResults": { + "subscriptionID": "32d2aec1-a6d2-46d9-900a-24124288cce1", + "queryName": "SimpleEventQuery", + "repository-x:ignoreFields": ["repository-x:hash", "example:captureID"], + "resultsBody": { + "eventList": [ + { + "type": "ObjectEvent", + "eventTime": "2019-10-21T14:45:00Z", + "recordTime": "2023-06-02T07:03:16.073Z", + "eventTimeZoneOffset": "+01:00", + "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "epcList": [ + "https://id.gs1.org/00/040123451111111127" + ], + "action": "OBSERVE", + "bizStep": "inspecting", + "disposition": "in_progress", + "readPoint": { + "id": "https://id.gs1.org/414/4012345000245/254/1" + }, + "repository-x:hash": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "example:captureID": "cc90bbe4-7f3a-4abd-b150-d4371c72a64f" + }, + { + "type": "ObjectEvent", + "eventTime": "2019-10-21T14:45:00Z", + "recordTime": "2023-06-02T07:03:16.073Z", + "eventTimeZoneOffset": "+01:00", + "eventID": "ni:///sha-256;b3d481c5590757ab8361a6cb0e924820feb3b61eb568e7d7703f21a96f76f51d?ver=CBV2.0", + "epcList": [ + "https://id.gs1.org/00/040123451111111127" + ], + "action": "OBSERVE", + "bizStep": "inspecting", + "disposition": "in_progress", + "readPoint": { + "id": "https://id.gs1.org/414/4012345000245/254/1" + } + } + ] + } + } + } + } diff --git a/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.prehashes b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.prehashes new file mode 100644 index 0000000..0055eb9 --- /dev/null +++ b/tests/examples/epcisQueryDocHavingEventWithIgnoreFields.prehashes @@ -0,0 +1,2 @@ +eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1 +eventType=ObjectEventeventTime=2019-10-21T14:45:00.000ZeventTimeZoneOffset=+01:00epcListepc=https://id.gs1.org/00/040123451111111127action=OBSERVEbizStep=https://ref.gs1.org/cbv/BizStep-inspectingdisposition=https://ref.gs1.org/cbv/Disp-in_progressreadPointid=https://id.gs1.org/414/4012345000245/254/1 diff --git a/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.hashes b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.hashes new file mode 100644 index 0000000..c441e4c --- /dev/null +++ b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.hashes @@ -0,0 +1,2 @@ +ni:///sha-256;2e3080ec5bfad98586d73f1943fae80e201613c1b2c0f084b3b8cd5976aaea5c?ver=CBV2.0 +ni:///sha-256;bfcf5b87dd396d4f7b266e33d4a969dc5250344ed0e47e85ce8e26d1adadad11?ver=CBV2.0 diff --git a/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.prehashes b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.prehashes new file mode 100644 index 0000000..bfcad9d --- /dev/null +++ b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.prehashes @@ -0,0 +1,2 @@ +eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField4=ABCD +eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField3=PQRS diff --git a/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.xml b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.xml new file mode 100644 index 0000000..f58b9c8 --- /dev/null +++ b/tests/examples/epcisXmlDocHavingEventWithIgnoreFields.xml @@ -0,0 +1,61 @@ + + + + + + + + + + 2024-06-08T10:58:56.591+02:00 + 2024-06-08T15:38:32.052+02:00 + +02:00 + + ADD + urn:epcglobal:cbv:bizstep:commissioning + + urn:epc:id:sgln:4054738.00005.0 + + + + + urn:epc:class:lgtin:4054739.099961.2015-03-30a + 150 + KGM + + + + 1234 + ABCD + + + 2024-06-08T10:58:56.591+02:00 + 2024-06-08T15:38:32.052+02:00 + +02:00 + + ADD + urn:epcglobal:cbv:bizstep:commissioning + + urn:epc:id:sgln:4054738.00005.0 + + + + + urn:epc:class:lgtin:4054739.099961.2015-03-30a + 150 + KGM + + + + 1234 + PQRS + + + + diff --git a/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.hashes b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.hashes new file mode 100644 index 0000000..4b18511 --- /dev/null +++ b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.hashes @@ -0,0 +1,2 @@ +ni:///sha-256;573a43267f95d2ddbe7e7487ad3ffb89e78ee8c72a296b868d9e5baeb8baef34?ver=CBV2.0 +ni:///sha-256;1e6c7bbfc2d3f744e99af09ad8216505b72f0ced383e391f92ab92674dc906c4?ver=CBV2.0 diff --git a/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.prehashes b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.prehashes new file mode 100644 index 0000000..f95f0e1 --- /dev/null +++ b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.prehashes @@ -0,0 +1,3 @@ +eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField4=ABC +eventType=ObjectEventeventTime=2024-06-08T08:58:56.591ZeventTimeZoneOffset=+02:00quantityListquantityElementepcClass=https://id.gs1.org/01/04054739999612/10/2015-03-30aquantity=150uom=KGMaction=ADDbizStep=https://ref.gs1.org/cbv/BizStep-commissioningreadPointid=https://id.gs1.org/414/4054738000050{https://ns.example.com/epcis/}testField3=XYZ + diff --git a/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.xml b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.xml new file mode 100644 index 0000000..b57a6c2 --- /dev/null +++ b/tests/examples/epcisXmlQueryDocHavingEventWithIgnoreFields.xml @@ -0,0 +1,62 @@ + + + + + + + SimpleEventQuery + + + + 2024-06-08T10:58:56.591+02:00 + 2024-06-08T15:38:32.052+02:00 + +02:00 + + ADD + urn:epcglobal:cbv:bizstep:commissioning + + urn:epc:id:sgln:4054738.00005.0 + + + + + urn:epc:class:lgtin:4054739.099961.2015-03-30a + 150 + KGM + + + + 123 + ABC + + + 2024-06-08T10:58:56.591+02:00 + 2024-06-08T15:38:32.052+02:00 + +02:00 + + ADD + urn:epcglobal:cbv:bizstep:commissioning + + urn:epc:id:sgln:4054738.00005.0 + + + + + urn:epc:class:lgtin:4054739.099961.2015-03-30a + 150 + KGM + + + + 123 + XYZ + + + +