From 3fd2868006a37dc3b467fa84fccad1f0599ad51f Mon Sep 17 00:00:00 2001 From: chiaberry Date: Wed, 10 Jan 2024 18:09:46 -0600 Subject: [PATCH 1/2] add the socrata step to sign specs --- ...d_knack_signs_work_order_specifications.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/dags/atd_knack_signs_work_order_specifications.py b/dags/atd_knack_signs_work_order_specifications.py index bdfe776..0736939 100644 --- a/dags/atd_knack_signs_work_order_specifications.py +++ b/dags/atd_knack_signs_work_order_specifications.py @@ -46,6 +46,18 @@ "opitem": "ArcGIS Online (AGOL) Scripts Publisher", "opfield": "production.password", }, + "SOCRATA_API_KEY_ID": { + "opitem": "Socrata Key ID, Secret, and Token", + "opfield": "socrata.apiKeyId", + }, + "SOCRATA_API_KEY_SECRET": { + "opitem": "Socrata Key ID, Secret, and Token", + "opfield": "socrata.apiKeySecret", + }, + "SOCRATA_APP_TOKEN": { + "opitem": "Socrata Key ID, Secret, and Token", + "opfield": "socrata.appToken", + }, } @@ -54,7 +66,7 @@ description="Publish sign work order specifications to Postgres, AGOL", default_args=DEFAULT_ARGS, schedule_interval="35 1 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None, - tags=["repo:atd-knack-services", "knack", "agol", "signs-markings"], + tags=["repo:atd-knack-services", "knack", "agol", "signs-markings", "socrata"], catchup=False, ) as dag: docker_image = "atddocker/atd-knack-services:production" @@ -68,6 +80,7 @@ t1 = DockerOperator( task_id="signs_asset_specs_to_postgrest", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_postgrest.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -79,6 +92,7 @@ t2 = DockerOperator( task_id="signs_asset_specs_to_agol", image=docker_image, + docker_conn_id="docker_default", auto_remove=True, command=f"./atd-knack-services/services/records_to_agol.py -a {app_name} -c {container} {date_filter_arg}", environment=env_vars, @@ -86,4 +100,15 @@ mount_tmp_dir=False, ) - date_filter_arg >> t1 >> t2 + t3 = DockerOperator( + task_id="signs_asset_specs_to_socrata", + image=docker_image, + docker_conn_id="docker_default", + auto_remove=True, + command=f"./atd-knack-services/services/records_to_socrata.py -a {app_name} -c {container} {date_filter_arg}", + environment=env_vars, + tty=True, + mount_tmp_dir=False, + ) + + date_filter_arg >> t1 >> t2 >> t3 From 7959400cbfbc36000760c35f0d9b57e633a0e868 Mon Sep 17 00:00:00 2001 From: chiaberry Date: Wed, 10 Jan 2024 18:24:43 -0600 Subject: [PATCH 2/2] add view information to description --- dags/atd_knack_signs_work_order_specifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/atd_knack_signs_work_order_specifications.py b/dags/atd_knack_signs_work_order_specifications.py index 0736939..085068d 100644 --- a/dags/atd_knack_signs_work_order_specifications.py +++ b/dags/atd_knack_signs_work_order_specifications.py @@ -63,7 +63,7 @@ with DAG( dag_id="atd_knack_signs_work_order_specifications", - description="Publish sign work order specifications to Postgres, AGOL", + description="Publish sign work order specifications (view_3106) to Postgres, AGOL", default_args=DEFAULT_ARGS, schedule_interval="35 1 * * *" if DEPLOYMENT_ENVIRONMENT == "production" else None, tags=["repo:atd-knack-services", "knack", "agol", "signs-markings", "socrata"],