From 4e98473d22b9dbe2ed66dec5c65f29982527375f Mon Sep 17 00:00:00 2001 From: keegansmith21 Date: Tue, 14 May 2024 16:53:21 +0800 Subject: [PATCH] Additions to onix WF --- dags/oaebu_workflows/onix_workflow/onix_workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/oaebu_workflows/onix_workflow/onix_workflow.py b/dags/oaebu_workflows/onix_workflow/onix_workflow.py index 45063070..1d2b9c93 100644 --- a/dags/oaebu_workflows/onix_workflow/onix_workflow.py +++ b/dags/oaebu_workflows/onix_workflow/onix_workflow.py @@ -35,7 +35,7 @@ from oaebu_workflows.airflow_pools import CrossrefEventsPool from oaebu_workflows.config import oaebu_user_agent_header, schema_folder as default_schema_folder, sql_folder -from oaebu_workflows.oaebu_partners import DataPartner, OaebuPartner, partner_from_str +from oaebu_workflows.oaebu_partners import DataPartner, OaebuPartner, partner_from_str, create_bespoke_data_partners from oaebu_workflows.onix_workflow.onix_work_aggregation import BookWorkAggregator, BookWorkFamilyAggregator from observatory_platform.airflow.airflow import on_failure_callback from observatory_platform.airflow.release import make_snapshot_date, set_task_state, SnapshotRelease @@ -308,6 +308,8 @@ def create_dag( metadata_partner = partner_from_str(metadata_partner, metadata_partner=True) data_partners = [partner_from_str(p) for p in data_partners] + if bespoke_data_partners: + data_partners.extend(create_bespoke_data_partners(bespoke_data_partners)) # Create pool for crossref API calls (if they don't exist) # Pools are necessary to throttle the maxiumum number of requests we can make per second and avoid 429 errors