fix(pipeline): deactivate unit test in prod

gip-inclusion · Jan 9, 2025 · 90c5aff · 90c5aff
1 parent 18001f1
commit 90c5aff
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 0 deletions.
diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml
@@ -34,6 +34,7 @@ x-airflow-common:
     AIRFLOW_VAR_DORA_API_TOKEN: ${AIRFLOW_VAR_DORA_API_TOKEN}
     AIRFLOW_VAR_DORA_API_URL: ${AIRFLOW_VAR_DORA_API_URL}
     AIRFLOW_VAR_EMPLOIS_API_TOKEN: ${AIRFLOW_VAR_EMPLOIS_API_TOKEN}
+    AIRFLOW_VAR_ENVIRONMENT: ${AIRFLOW_VAR_ENVIRONMENT}
     AIRFLOW_VAR_FREDO_API_TOKEN: ${AIRFLOW_VAR_FREDO_API_TOKEN}
     AIRFLOW_VAR_FT_API_TOKEN: ${AIRFLOW_VAR_FT_API_TOKEN}
     AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY: ${AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY}

diff --git a/deployment/main.tf b/deployment/main.tf
@@ -245,6 +245,7 @@ resource "null_resource" "up" {
     AIRFLOW_VAR_DORA_API_TOKEN='${var.dora_api_token}'
     AIRFLOW_VAR_DORA_API_URL='${var.dora_api_url}'
     AIRFLOW_VAR_EMPLOIS_API_TOKEN='${var.emplois_api_token}'
+    AIRFLOW_VAR_ENVIRONMENT='${var.environment}'
     AIRFLOW_VAR_FREDO_API_TOKEN='${var.fredo_api_token}'
     AIRFLOW_VAR_FT_API_TOKEN='${var.ft_api_token}'
     AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY='${var.mes_aides_airtable_key}'

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -36,6 +36,7 @@ x-airflow-common:
     AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY: ${AIRFLOW_VAR_MES_AIDES_AIRTABLE_KEY}
     AIRFLOW_VAR_SOLIGUIDE_API_TOKEN: ${AIRFLOW_VAR_SOLIGUIDE_API_TOKEN}
     AIRFLOW_VAR_TWOCAPTCHA_API_KEY: ${AIRFLOW_VAR_TWOCAPTCHA_API_KEY}
+    AIRFLOW_VAR_ENVIRONMENT: test
 
   volumes:
     - ./pipeline/dbt:/opt/airflow/dbt

diff --git a/pipeline/dags/dag_utils/dbt.py b/pipeline/dags/dag_utils/dbt.py
@@ -17,6 +17,7 @@ def dbt_operator_factory(
     select: Optional[str] = None,
     exclude: Optional[str] = None,
     trigger_rule: TriggerRule = TriggerRule.ALL_SUCCESS,
+    exclude_unit_tests: bool = False,
 ) -> bash.BashOperator:
     """A basic factory for bash operators operating dbt commands."""
 
@@ -28,6 +29,8 @@ def dbt_operator_factory(
     if dbt_vars is not None:
         dbt_vars = json.dumps(dbt_vars)
         dbt_args += f" --vars '{dbt_vars}'"
+    if exclude_unit_tests:
+        dbt_args += " --exclude-resource-type unit_test"
 
     return bash.BashOperator(
         task_id=task_id,
@@ -96,6 +99,7 @@ def get_staging_tasks():
 
 
 def get_intermediate_tasks():
+    exclude_unit_tests = True if Variable.get("ENVIRONMENT", None) == "prod" else False
     return dbt_operator_factory(
         task_id="dbt_build_intermediate",
         command="build",
@@ -128,4 +132,5 @@ def get_intermediate_tasks():
             ]
         ),
         trigger_rule=TriggerRule.ALL_DONE,
+        exclude_unit_tests=exclude_unit_tests,
     )