From aea282fa6d680f003983160946b37ff50f8aa8fc Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Wed, 20 Dec 2023 14:45:44 -0600 Subject: [PATCH 1/4] first pass at DAG script --- dags/vz_populate_cr3_metadata.py | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 dags/vz_populate_cr3_metadata.py diff --git a/dags/vz_populate_cr3_metadata.py b/dags/vz_populate_cr3_metadata.py new file mode 100644 index 0000000..c2e54a3 --- /dev/null +++ b/dags/vz_populate_cr3_metadata.py @@ -0,0 +1,74 @@ +import os +from pendulum import datetime +from airflow.decorators import dag, task +from airflow.operators.docker_operator import DockerOperator +from utils.slack_operator import task_fail_slack_alert + +default_args = { + "owner": "airflow", + "depends_on_past": False, + "start_date": datetime(2019, 1, 1, tz="America/Chicago"), + "email_on_failure": False, + "email_on_retry": False, + "retries": 0, + "execution_timeout": duration(minutes=5), + "on_failure_callback": task_fail_slack_alert, +} + +DEPLOYMENT_ENVIRONMENT = os.getenv("ENVIRONMENT", "development") + +REQUIRED_SECRETS = { + "AWS_BUCKET_NAME": { + "opitem": "Vision Zero CR3 Metadata Extraction", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.AWS Bucket for CR3s", + }, + "AWS_BUCKET_ENVIRONMENT": { + "opitem": "Vision Zero CR3 Metadata Extraction", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.AWS S3 Path for CR3s", + }, + "HASURA_ENDPOINT": { + "opitem": "Vision Zero graphql-engine Endpoints", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.GraphQL Endpoint", + }, + "HASURA_ADMIN_KEY": { + "opitem": "Vision Zero graphql-engine Endpoints", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.Admin Key", + }, +} + + +@dag( + dag_id="vz-populate-cr3-metadata", + description="A DAG which populates CR3 metadata in the VZDB", + schedule_interval="*/5 8-10 * * *" + if DEPLOYMENT_ENVIRONMENT == "production" + else None, + catchup=False, + tags=["repo:atd-vz-data", "vision-zero", "ocr", "cr3"], + default_args=default_args, +) +def populate_cr3_metadata(): + @task( + task_id="get_env_vars", + execution_timeout=duration(seconds=30), + ) + def get_env_vars(): + from utils.onepassword import load_dict + + return load_dict(REQUIRED_SECRETS) + + env_vars = get_env_vars() + + DockerOperator( + task_id="run_cr3_metadata_population", + environment=env_vars, + image="atddocker/vz-cr3-metadata-pdfs:production", + auto_remove=True, + #entrypoint=["/entrypoint.sh"], + #command=["ems"], + tty=True, + force_pull=True, + ) + + +populate_cr3_metadata() From 96791ea69be5797507ea823652fbdad787817e05 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 21 Dec 2023 11:33:57 -0600 Subject: [PATCH 2/4] additional credentials --- dags/vz_populate_cr3_metadata.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/dags/vz_populate_cr3_metadata.py b/dags/vz_populate_cr3_metadata.py index c2e54a3..f85b52a 100644 --- a/dags/vz_populate_cr3_metadata.py +++ b/dags/vz_populate_cr3_metadata.py @@ -1,5 +1,5 @@ import os -from pendulum import datetime +from pendulum import datetime, duration from airflow.decorators import dag, task from airflow.operators.docker_operator import DockerOperator from utils.slack_operator import task_fail_slack_alert @@ -34,6 +34,14 @@ "opitem": "Vision Zero graphql-engine Endpoints", "opfield": f"{DEPLOYMENT_ENVIRONMENT}.Admin Key", }, + "AWS_ACCESS_KEY_ID": { + "opitem": "Vision Zero CRIS Import", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.AWS Access key", + }, + "AWS_SECRET_ACCESS_KEY": { + "opitem": "Vision Zero CRIS Import", + "opfield": f"{DEPLOYMENT_ENVIRONMENT}.AWS Secret key", + }, } @@ -55,7 +63,11 @@ def populate_cr3_metadata(): def get_env_vars(): from utils.onepassword import load_dict - return load_dict(REQUIRED_SECRETS) + answers = load_dict(REQUIRED_SECRETS) + print(answers) + return answers + + # return load_dict(REQUIRED_SECRETS) env_vars = get_env_vars() @@ -64,7 +76,7 @@ def get_env_vars(): environment=env_vars, image="atddocker/vz-cr3-metadata-pdfs:production", auto_remove=True, - #entrypoint=["/entrypoint.sh"], + entrypoint=["/app/populate_cr3_file_metadata.py"], #command=["ems"], tty=True, force_pull=True, From 007d41fe0fd2e83ff9b02d48e3d89ee83c94e60f Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 21 Dec 2023 11:40:29 -0600 Subject: [PATCH 3/4] tidy up this debugging construct --- dags/vz_populate_cr3_metadata.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dags/vz_populate_cr3_metadata.py b/dags/vz_populate_cr3_metadata.py index f85b52a..426eb52 100644 --- a/dags/vz_populate_cr3_metadata.py +++ b/dags/vz_populate_cr3_metadata.py @@ -62,12 +62,7 @@ def populate_cr3_metadata(): ) def get_env_vars(): from utils.onepassword import load_dict - - answers = load_dict(REQUIRED_SECRETS) - print(answers) - return answers - - # return load_dict(REQUIRED_SECRETS) + return load_dict(REQUIRED_SECRETS) env_vars = get_env_vars() From 41fa87eed128d91f24875906fffc0c4306d9130e Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 21 Dec 2023 11:41:47 -0600 Subject: [PATCH 4/4] remove comment --- dags/vz_populate_cr3_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/vz_populate_cr3_metadata.py b/dags/vz_populate_cr3_metadata.py index 426eb52..b2470a2 100644 --- a/dags/vz_populate_cr3_metadata.py +++ b/dags/vz_populate_cr3_metadata.py @@ -62,6 +62,7 @@ def populate_cr3_metadata(): ) def get_env_vars(): from utils.onepassword import load_dict + return load_dict(REQUIRED_SECRETS) env_vars = get_env_vars() @@ -72,7 +73,6 @@ def get_env_vars(): image="atddocker/vz-cr3-metadata-pdfs:production", auto_remove=True, entrypoint=["/app/populate_cr3_file_metadata.py"], - #command=["ems"], tty=True, force_pull=True, )