Merge pull request #246 from cityofaustin/ch-parking-etl-dates

Parking ETL: Remove PARD account and set 14 days as default
cityofaustin · Aug 23, 2024 · cf3b94f · cf3b94f
2 parents e3c2e05 + 5d0c9f1
commit cf3b94f
Showing 1 changed file with 2 additions and 34 deletions.
diff --git a/dags/atd_parking_data.py b/dags/atd_parking_data.py
@@ -152,8 +152,8 @@
 ) as dag:
     env_vars = get_env_vars_task(REQUIRED_SECRETS)
 
-    # default to the last 7 days of transactions but if we haven't ran it before, collect the last two weeks.
-    prev_exec = "{{ (prev_start_date_success - macros.timedelta(days=7)).strftime('%Y-%m-%d') if prev_start_date_success else (execution_date - macros.timedelta(days=14)).strftime('%Y-%m-%d')}}"
+    # default to the last 14 days of transactions
+    prev_exec = "{{ (prev_start_date_success - macros.timedelta(days=14)).strftime('%Y-%m-%d') if prev_start_date_success else (execution_date - macros.timedelta(days=14)).strftime('%Y-%m-%d')}}"
 
     docker_tasks = []
 
@@ -189,22 +189,6 @@
         )
     )
 
-    docker_tasks.append(
-        DockerOperator(
-            task_id="smartfolio_pard_payments",
-            image=docker_image,
-            docker_conn_id="docker_default",
-            command=f"python txn_history.py -v --report payments --user pard --env prod --start {prev_exec}",
-            api_version="auto",
-            auto_remove=True,
-            environment=env_vars,
-            tty=True,
-            force_pull=False,
-            retries=3,
-            retry_delay=duration(seconds=60),
-        )
-    )
-
     docker_tasks.append(
         DockerOperator(
             task_id="passport_transactions",
@@ -269,22 +253,6 @@
         )
     )
 
-    docker_tasks.append(
-        DockerOperator(
-            task_id="process_pard_smartfolio_payments",
-            image=docker_image,
-            docker_conn_id="docker_default",
-            command=f"python payments_s3.py --user pard --lastmonth True",
-            api_version="auto",
-            auto_remove=True,
-            environment=env_vars,
-            tty=True,
-            force_pull=False,
-            retries=3,
-            retry_delay=duration(seconds=60),
-        )
-    )
-
     docker_tasks.append(
         DockerOperator(
             task_id="process_passport_transactions",