Improvements for Zyp/Moksha error reporting and MongoDB $date.$numberLong values

CHANGES.md

@@ -3,6 +3,8 @@
 ## Unreleased
 - DynamoDB: Change CrateDB data model to use (`pk`, `data`, `aux`) columns
   Attention: This is a breaking change.
+- MongoDB: Handle too large `$date.$numberLong` values gracefully
+- Zyp/Moksha: Improve error reporting when rule evaluation fails
 
 ## 2024/09/26 v0.0.19
 - DynamoDB CDC: Fix `MODIFY` operation by propagating `NewImage` fully

src/commons_codec/transform/mongodb.py

@@ -123,7 +123,11 @@ def decode_extended_json(self, value: t.Dict[str, t.Any]) -> t.Any:
         type_ = next(iter(value))  # Get key of first item in dictionary.
         is_date_numberlong = type_ == "$date" and "$numberLong" in value["$date"]
         if is_date_numberlong:
-            out = dt.datetime.fromtimestamp(int(value["$date"]["$numberLong"]) / 1000, tz=dt.timezone.utc)
+            try:
+                out = dt.datetime.fromtimestamp(int(value["$date"]["$numberLong"]) / 1000, tz=dt.timezone.utc)
+            except ValueError as ex:
+                logger.error(f"Decoding legacy timestamp failed: {ex}. value={value}")
+                out = 0
         else:
             out = object_hook(value)
 

src/zyp/model/moksha.py

@@ -22,11 +22,14 @@ class MokshaRule:
     disabled: t.Optional[bool] = False
 
     def compile(self):
-        return MokshaRuntimeRule(self.type, compile_expression(self.type, self.expression), disabled=self.disabled)
+        return MokshaRuntimeRule(
+            self, self.type, compile_expression(self.type, self.expression), disabled=self.disabled
+        )
 
 
 @define
 class MokshaRuntimeRule:
+    source: MokshaRule
     type: str
     transformer: MokshaTransformer
     disabled: t.Optional[bool] = False
@@ -76,10 +79,11 @@ def apply(self, data: t.Any) -> t.Any:
         for rule in self._runtime_rules:
             try:
                 data = rule.evaluate(data)
-            except Exception:
-                logger.exception(f"Error evaluating rule: {rule}")
-                if isinstance(data, map):
-                    data = list(data)
-                logger.debug(f"Error payload:\n{data}")
+            except Exception as ex:
+                logger.error(f"Error evaluating rule: {ex}. Expression: {rule.source.expression}")
+                if logger.level is logging.DEBUG:
+                    if isinstance(data, map):
+                        data = list(data)
+                    logger.debug(f"Error payload:\n{data}")
                 raise
         return data

tests/transform/mongodb/data.py

@@ -58,7 +58,10 @@
         "code_bytes": {"$code": "ab\u0000ab\u0000"},
         "code_scope": {"$code": "abab", "$scope": {"x": {"$numberInt": "42"}}},
         "date_iso8601": {"$date": "2015-09-23T10:32:42.33Z"},
-        "date_numberlong": {"$date": {"$numberLong": "1356351330000"}},
+        "date_numberlong_valid": {"$date": {"$numberLong": "1356351330000"}},
+        "date_numberlong_invalid": {
+            "$date": {"$numberLong": "-9223372036854775808"}
+        },  # year -292275055 is out of range
         "dbref": {
             "$id": {"$oid": "56027fcae4b09385a85f9344"},
             "$ref": "foo",
@@ -169,7 +172,8 @@
             },
         },
         "date_iso8601": 1443004362000,
-        "date_numberlong": 1356351330000,
+        "date_numberlong_valid": 1356351330000,
+        "date_numberlong_invalid": 0,
         "dbref": {
             "$id": "56027fcae4b09385a85f9344",
             "$ref": "foo",

tests/zyp/moksha/test_model.py

@@ -1,3 +1,4 @@
+import logging
 import re
 
 import pytest
@@ -50,22 +51,27 @@ def test_moksha_transformation_success_jq():
 
 
 def test_moksha_transformation_error_jq_scalar(caplog):
+    logging.getLogger("zyp.model.moksha").setLevel(logging.DEBUG)
     moksha = MokshaTransformation().jq(". /= 100")
     with pytest.raises(ValueError) as ex:
         moksha.apply("foo")
     assert ex.match(re.escape('string ("foo") and number (100) cannot be divided'))
 
-    assert "Error evaluating rule: MokshaRuntimeRule(type='jq'" in caplog.text
+    assert (
+        'Error evaluating rule: string ("foo") and number (100) cannot be divided. Expression: . /= 100'
+        in caplog.messages
+    )
     assert "Error payload:\nfoo" in caplog.messages
 
 
 def test_moksha_transformation_error_jq_map(caplog):
+    logging.getLogger("zyp.model.moksha").setLevel(logging.DEBUG)
     moksha = MokshaTransformation().jq(".foo")
     with pytest.raises(ValueError) as ex:
         moksha.apply(map(lambda x: x, ["foo"]))  # noqa: C417
     assert ex.match(re.escape('Cannot index array with string "foo"'))
 
-    assert "Error evaluating rule: MokshaRuntimeRule(type='jq'" in caplog.text
+    assert 'Error evaluating rule: Cannot index array with string "foo". Expression: .foo' in caplog.messages
     assert "Error payload:\n[]" in caplog.messages