From 21a0144b21bb29fa3c0d01f98fd66b0c21ad12ea Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 25 Sep 2024 21:31:51 +0200 Subject: [PATCH] MongoDB: Configure `MongoDBCrateDBConverter` for updating commons-codec commons-codec 0.0.18 provides configuration capabilities about how to decode timestamp values. CTK currently uses this configuration: MongoDBCrateDBConverter( timestamp_to_epoch=True, timestamp_use_milliseconds=True, ) --- CHANGES.md | 1 + cratedb_toolkit/io/mongodb/copy.py | 9 +++++++-- pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ffb46c8..79382c7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## Unreleased +- MongoDB: Configure `MongoDBCrateDBConverter` after updating to commons-codec 0.0.18 ## 2024/09/22 v0.0.25 - Table Loader: Improved conditional handling of "transformation" parameter diff --git a/cratedb_toolkit/io/mongodb/copy.py b/cratedb_toolkit/io/mongodb/copy.py index febfbe4..27ce6ba 100644 --- a/cratedb_toolkit/io/mongodb/copy.py +++ b/cratedb_toolkit/io/mongodb/copy.py @@ -57,9 +57,14 @@ def __init__( ) try: transformation = tm.project.get(address=address) + logger.info(f"Applying transformation to: {address}") except KeyError: - pass - self.converter = MongoDBCrateDBConverter(transformation=transformation) + logger.warning(f"No transformation found for: {address}") + self.converter = MongoDBCrateDBConverter( + timestamp_to_epoch=True, + timestamp_use_milliseconds=True, + transformation=transformation, + ) self.translator = MongoDBFullLoadTranslator(table_name=self.cratedb_table, converter=self.converter) self.on_error = on_error diff --git a/pyproject.toml b/pyproject.toml index 58705ed..fc1bdb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,7 +167,7 @@ kinesis = [ "lorrystream[carabas]>=0.0.6", ] mongodb = [ - "commons-codec[mongodb,zyp]>=0.0.17", + "commons-codec[mongodb,zyp]>=0.0.18", "cratedb-toolkit[io]", "orjson<4,>=3.3.1", "pymongo<5,>=3.10.1",