Merge pull request #181 from cityofaustin/13946-school-beacons-edp

Add dag for internal School Zone Beacons dataset
cityofaustin · Dec 18, 2023 · dfce3fd · dfce3fd
2 parents d4edec5 + b9bf537
commit dfce3fd
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 10 deletions.
diff --git a/dags/atd_kits_sig_stat_pub.py b/dags/atd_kits_sig_stat_pub.py
@@ -55,7 +55,7 @@
 
 with DAG(
     dag_id=f"atd_kits_sig_stat_pub",
-    description="Fetch signal flash statuses KITS and publish to Socata",
+    description="Fetch signal flash statuses KITS and publish to Socrata",
     default_args=DEFAULT_ARGS,
     schedule_interval="*/5 * * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
     tags=["repo:atd-kits", "socrata", "kits"],

diff --git a/dags/atd_knack_banner.py b/dags/atd_knack_banner.py
@@ -17,7 +17,7 @@
     "email_on_failure": False,
     "email_on_retry": False,
     "retries": 0,
-    "retry_delay": duration(minutes=5),
+    "execution_timeout": duration(minutes=10),
     "on_failure_callback": task_fail_slack_alert,
 }
 
@@ -47,7 +47,6 @@
     description="Update knack HR app based on records in Banner and CTM",
     default_args=DEFAULT_ARGS,
     schedule_interval="45 7 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
-    dagrun_timeout=duration(minutes=30),
     tags=["repo:atd-knack-banner", "knack"],
     catchup=False,
 ) as dag:

diff --git a/dags/atd_knack_corridor_retiming.py b/dags/atd_knack_corridor_retiming.py
@@ -55,7 +55,7 @@
 
 with DAG(
     dag_id=f"atd_knack_corridor_retiming",
-    description="Load corridor retiming data from Knack to Postgrest to Socata",
+    description="Load corridor retiming data from Knack to Postgrest to Socrata",
     default_args=DEFAULT_ARGS,
     schedule_interval="45 11 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
     tags=["repo:atd-knack-services", "knack", "socrata", "data-tracker"],

diff --git a/dags/atd_knack_metrobike_kiosks.py b/dags/atd_knack_metrobike_kiosks.py
@@ -63,7 +63,7 @@
 
 with DAG(
     dag_id="atd_knack_metrobike_kiosks",
-    description="Load metrobike kiosk records from Knack to Postgrest to AGOL and Socata",
+    description="Load metrobike kiosk records from Knack to Postgrest to AGOL and Socrata",
     default_args=DEFAULT_ARGS,
     schedule_interval="5 2 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
     tags=["repo:atd-knack-services", "knack", "socrata", "agol", "data-tracker"],

diff --git a/dags/atd_knack_school_beacons.py b/dags/atd_knack_school_beacons.py
@@ -46,15 +46,27 @@
         "opitem": "ArcGIS Online (AGOL) Scripts Publisher",
         "opfield": "production.password",
     },
+    "SOCRATA_API_KEY_ID": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.apiKeyId",
+    },
+    "SOCRATA_API_KEY_SECRET": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.apiKeySecret",
+    },
+    "SOCRATA_APP_TOKEN": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.appToken",
+    },
 }
 
 
 with DAG(
     dag_id="atd_knack_school_beacons",
-    description="Load school beacons (view_3086) records from Knack to Postgrest to AGOL",
+    description="Load school beacons (view_3086) records from Knack to Postgrest to AGOL and Socrata",
     default_args=DEFAULT_ARGS,
     schedule_interval="25 4 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
-    tags=["repo:atd-knack-services", "knack", "agol", "data-tracker"],
+    tags=["repo:atd-knack-services", "knack", "agol", "data-tracker", "socrata"],
     catchup=False,
 ) as dag:
     docker_image = "atddocker/atd-knack-services:production"
@@ -68,6 +80,7 @@
     t1 = DockerOperator(
         task_id="atd_knack_school_beacons_to_postgrest",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
@@ -79,11 +92,23 @@
     t2 = DockerOperator(
         task_id="atd_knack_school_beacons_to_agol",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}",
         environment=env_vars,
         tty=True,
         mount_tmp_dir=False,
     )
 
-    date_filter_arg >> t1 >> t2
+    t3 = DockerOperator(
+        task_id="atd_knack_school_beacons_to_socrata",
+        image=docker_image,
+        docker_conn_id="docker_default",
+        auto_remove=True,
+        command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}",
+        environment=env_vars,
+        tty=True,
+        mount_tmp_dir=False,
+    )
+
+    date_filter_arg >> t1 >> t2 >> t3
diff --git a/dags/atd_knack_signals.py b/dags/atd_knack_signals.py
@@ -63,7 +63,7 @@
 
 with DAG(
     dag_id=f"atd_knack_signals",
-    description="Load signals (view_197) records from Knack to Postgrest to AGOL and Socata",
+    description="Load signals (view_197) records from Knack to Postgrest to AGOL and Socrata",
     default_args=DEFAULT_ARGS,
     schedule_interval="1 */2 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None,
     tags=["repo:atd-knack-services", "knack", "socrata", "agol", "data-tracker"],

diff --git a/dags/atd_signal_comms.py b/dags/atd_signal_comms.py
@@ -149,7 +149,7 @@ def get_start_date(**context):
     )
 
     cameras_socrata = DockerOperator(
-        task_id="socata_pub_cameras",
+        task_id="socrata_pub_cameras",
         image=docker_image,
         auto_remove=True,
         command=f"python atd-signal-comms/socrata_pub.py camera --start {start_date} -v --env {DEPLOYMENT_ENVIRONMENT}",