Limit to 2 alter_data_batches concurrently (#5125)

* Limit to 10 alter_data_batches concurrently * Reduce to 2 concurrent batches
WordPress · Oct 31, 2024 · 071e422 · 071e422
1 parent ea7766d
commit 071e422
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/catalog/dags/data_refresh/alter_data.py b/catalog/dags/data_refresh/alter_data.py
@@ -124,7 +124,7 @@ def get_alter_batches(
     return [(x, x + batch_size - 1) for x in range(start, stop, batch_size)]
 
 
-@task
+@task(max_active_tis_per_dagrun=2)
 def alter_data_batch(
     batch: tuple[int, int],
     temp_table: str,
@@ -203,7 +203,8 @@ def alter_table_data(
     )
 
     alter_data = alter_data_batch.partial(
-        temp_table=temp_table, postgres_conn_id=postgres_conn_id
+        temp_table=temp_table,
+        postgres_conn_id=postgres_conn_id,
     ).expand(batch=batches)
 
     report(alter_data)