update the path to delted and updated marc files. (#1641)

tulibraries · Oct 10, 2024 · 43e7253 · 43e7253
1 parent ac07520
commit 43e7253
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 9 deletions.
diff --git a/cob_datapipeline/catalog_preproduction_oai_harvest_dag.py b/cob_datapipeline/catalog_preproduction_oai_harvest_dag.py
@@ -176,8 +176,8 @@
 LIST_UPDATED_FILES = S3ListOperator(
         task_id="list_updated_files",
         bucket=AIRFLOW_DATA_BUCKET,
-        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/new-updated",
-        delimiter="/",
+        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/new-updated/",
+        delimiter="",
         aws_conn_id=AIRFLOW_S3.conn_id,
         trigger_rule="none_failed_min_one_success",
         )
@@ -206,8 +206,8 @@
 LIST_DELETED_FILES = S3ListOperator(
         task_id="list_deleted_files",
         bucket=AIRFLOW_DATA_BUCKET,
-        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/deleted",
-        delimiter="/",
+        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/deleted/",
+        delimiter="",
         aws_conn_id=AIRFLOW_S3.conn_id,
         trigger_rule="none_failed_min_one_success",
         )

diff --git a/cob_datapipeline/catalog_production_oai_harvest_dag.py b/cob_datapipeline/catalog_production_oai_harvest_dag.py
@@ -168,8 +168,8 @@
 LIST_UPDATED_FILES = S3ListOperator(
         task_id="list_updated_files",
         bucket=AIRFLOW_DATA_BUCKET,
-        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/new-updated",
-        delimiter="/",
+        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/new-updated/",
+        delimiter="",
         aws_conn_id=AIRFLOW_S3.conn_id,
         trigger_rule="none_failed_min_one_success",
         )
@@ -197,8 +197,8 @@
 LIST_DELETED_FILES = S3ListOperator(
         task_id="list_deleted_files",
         bucket=AIRFLOW_DATA_BUCKET,
-        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/deleted",
-        delimiter="/",
+        prefix=DAG.dag_id + f"/{ S3_NAME_SPACE }/deleted/",
+        delimiter="",
         aws_conn_id=AIRFLOW_S3.conn_id,
         trigger_rule="none_failed_min_one_success",
         )
@@ -210,7 +210,6 @@
         "AWS_ACCESS_KEY_ID": AIRFLOW_S3.login,
         "AWS_SECRET_ACCESS_KEY": AIRFLOW_S3.password,
         "BUCKET": AIRFLOW_DATA_BUCKET,
-        "FOLDER": DAG.dag_id + f"/{ S3_NAME_SPACE }/deleted",
         "GIT_BRANCH": COB_INDEX_VERSION,
         "HOME": AIRFLOW_USER_HOME,
         "LATEST_RELEASE": "false",