Merge pull request #1206 from DataJunction/DJ-108

Add lookback window to Materialization calls.
DataJunction · Oct 17, 2024 · c845c8a · c845c8a
2 parents 9ed081f + 0df280b
commit c845c8a
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 0 deletions.
diff --git a/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py b/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py
@@ -92,6 +92,7 @@ def schedule(
                 partitions=temporal_partition + categorical_partitions,
                 job=materialization.job,
                 strategy=materialization.strategy,
+                lookback_window=cube_config.lookback_window,
             ),
             request_headers=request_headers,
         )

diff --git a/datajunction-server/datajunction_server/models/materialization.py b/datajunction-server/datajunction_server/models/materialization.py
@@ -91,6 +91,7 @@ class GenericMaterializationInput(BaseModel):
     spark_conf: Optional[Dict] = None
     partitions: Optional[List[Dict]] = None
     columns: List[ColumnMetadata]
+    lookback_window: Optional[str] = "1 DAY"
 
 
 class DruidMaterializationInput(GenericMaterializationInput):

diff --git a/datajunction-server/tests/service_clients_test.py b/datajunction-server/tests/service_clients_test.py
@@ -420,6 +420,7 @@ def test_query_service_client_materialize(self, mocker: MockerFixture) -> None:
                 "spark_conf": {},
                 "upstream_tables": ["default.hard_hats"],
                 "columns": [],
+                "lookback_window": "1 DAY",
             },
             headers=ANY,
         )
@@ -544,6 +545,7 @@ def test_materialize(self, mocker: MockerFixture) -> None:
             json={
                 "name": "default",
                 "job": "SparkSqlMaterializationJob",
+                "lookback_window": "1 DAY",
                 "strategy": "full",
                 "node_name": "default.hard_hat",
                 "node_version": "v1",