Merge pull request #192 from cityofaustin/ch-backup-active-permits

Backing up active ROW permits log
cityofaustin · Dec 21, 2023 · 03d330b · 03d330b
2 parents dfce3fd + 75450e5
commit 03d330b
Showing 1 changed file with 51 additions and 1 deletion.
diff --git a/dags/atd_executive_dashboard_row_active_permits_logging.py b/dags/atd_executive_dashboard_row_active_permits_logging.py
@@ -1,5 +1,8 @@
+# test locally with: docker compose run --rm airflow-cli dags test atd_executive_dashboard_row_active_permits_logging
+
 import os
 
+from airflow.decorators import task
 from airflow.models import DAG
 from airflow.operators.docker_operator import DockerOperator
 from pendulum import datetime, duration, now
@@ -88,6 +91,36 @@
     },
 }
 
+SECRETS_SOCRATA_BACKUP = {
+    "SOCRATA_API_KEY_ID": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.apiKeyId",
+    },
+    "SOCRATA_API_KEY_SECRET": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.apiKeySecret",
+    },
+    "SOCRATA_APP_TOKEN": {
+        "opitem": "Socrata Key ID, Secret, and Token",
+        "opfield": "socrata.appToken",
+    },
+    "AWS_ACCESS_ID": {
+        "opitem": "Socrata Dataset Backups S3 Bucket",
+        "opfield": "production.AWS Access Key",
+    },
+    "AWS_SECRET_ACCESS_KEY": {
+        "opitem": "Socrata Dataset Backups S3 Bucket",
+        "opfield": "production.AWS Secret Access Key",
+    },
+    "BUCKET": {
+        "opitem": "Socrata Dataset Backups S3 Bucket",
+        "opfield": "production.Bucket",
+    },
+}
+
+@task
+def get_dataset_id(env_vars):
+    return env_vars['ACTIVE_DATASET']
 
 with DAG(
     dag_id="atd_executive_dashboard_row_active_permits_logging",
@@ -98,12 +131,16 @@
     catchup=False,
 ) as dag:
     docker_image = "atddocker/atd-executive-dashboard:production"
+    docker_image_2 = "atddocker/atd-knack-services:production"
 
     env_vars = get_env_vars_task(REQUIRED_SECRETS)
+    env_vars_2 = get_env_vars_task(SECRETS_SOCRATA_BACKUP)
+    dataset_id = get_dataset_id(env_vars)
 
     t1 = DockerOperator(
         task_id="active_permits_logging",
         image=docker_image,
+        docker_conn_id="docker_default",
         auto_remove=True,
         command=f"python active_permits_logging.py",
         environment=env_vars,
@@ -114,4 +151,17 @@
         retry_delay=duration(seconds=60),
     )
 
-    t1
+    t2 = DockerOperator(
+        task_id="backup_active_permits",
+        image=docker_image_2,
+        docker_conn_id="docker_default",
+        auto_remove=True,
+        command=f"./atd-knack-services/services/backup_socrata.py --dataset {dataset_id}",
+        environment=env_vars_2,
+        tty=True,
+        force_pull=True,
+        mount_tmp_dir=False,
+    )
+
+
+    t1 >> t2