From 67d80e034b4b8534eaba4810d6e19850cb26bf76 Mon Sep 17 00:00:00 2001
From: Hamza Azelmat <azelmat.hamza@gmail.com>
Date: Mon, 12 Aug 2024 16:24:43 +0100
Subject: [PATCH] Add a high number of retries and a short delay to retry on
 read tasks (#766)

* add columns for wiriting to displayed acteur

* add retry
---
 dags/create_final_actors.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/dags/create_final_actors.py b/dags/create_final_actors.py
index 444a0f6d..5d680fef 100644
--- a/dags/create_final_actors.py
+++ b/dags/create_final_actors.py
@@ -335,6 +335,12 @@ def _merge_acteurs_many2many_relationship(
     "retry_delay": timedelta(minutes=5),
 }
 
+
+# Retry settings for reading tasks
+read_retry_count = 5
+read_retry_interval = timedelta(minutes=2)
+
+
 dag = DAG(
     utils.get_dag_name(__file__, "apply_adresse_corrections"),
     default_args=default_args,
@@ -349,6 +355,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_acteur"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_ps = PythonOperator(
@@ -356,6 +364,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_propositionservice"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_revision_actor = PythonOperator(
@@ -363,6 +373,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_revisionacteur"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_revision_ps = PythonOperator(
@@ -370,6 +382,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_revisionpropositionservice"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_revision_sc = PythonOperator(
@@ -377,6 +391,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_revisionpropositionservice_sous_categories"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_sc = PythonOperator(
@@ -384,6 +400,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_propositionservice_sous_categories"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_acteur_labels = PythonOperator(
@@ -391,6 +409,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_acteur_labels"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_acteur_acteur_services = PythonOperator(
@@ -398,6 +418,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_acteur_acteur_services"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_revisionacteur_labels = PythonOperator(
@@ -405,6 +427,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_revisionacteur_labels"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 
 read_revisionacteur_acteur_services = PythonOperator(
@@ -412,6 +436,8 @@ def _merge_acteurs_many2many_relationship(
     python_callable=read_data_from_postgres,
     op_kwargs={"table_name": "qfdmo_revisionacteur_acteur_services"},
     dag=dag,
+    retries=read_retry_count,
+    retry_delay=read_retry_interval,
 )
 # qfdmo_revisionacteur_acteur_services