feat(pipeline) : integration of the imilo source (int))

int addition

int additions

int

fix : add the necessary imilo annotations for quality

+ little bug in the int_quality_stats because
  - the iteration is not technically over (there still is
    imilo_offres_structures)
  - but the next one is not a qualifying source (no kind) so
    the iteration fails
  - switch to loop.first and prepend UNION ALL to avoid any further issue

int additions

pipeline/dbt/models/intermediate/int__union_adresses.sql

@@ -10,6 +10,7 @@ WITH adresses AS (
                 ref('int_finess__adresses'),
                 ref('int_france_travail__adresses'),
                 ref('int_fredo__adresses'),
+                ref('int_imilo__adresses'),
                 ref('int_mediation_numerique__adresses'),
                 ref('int_mes_aides__adresses'),
                 ref('int_monenfant__adresses'),

pipeline/dbt/models/intermediate/int__union_services.sql

@@ -7,6 +7,7 @@ WITH services AS (
                 ref('int_dora__services'),
                 ref('int_france_travail__services'),
                 ref('int_fredo__services'),
+                ref('int_imilo__services'),
                 ref('int_mediation_numerique__services'),
                 ref('int_mes_aides__services'),
                 ref('int_monenfant__services'),

pipeline/dbt/models/intermediate/int__union_structures.sql

@@ -10,6 +10,7 @@ WITH structures AS (
                 ref('int_finess__structures'),
                 ref('int_france_travail__structures'),
                 ref('int_fredo__structures'),
+                ref('int_imilo__structures'),
                 ref('int_mediation_numerique__structures'),
                 ref('int_mes_aides__structures'),
                 ref('int_monenfant__structures'),

pipeline/dbt/models/intermediate/quality/_quality_models.yml

@@ -23,6 +23,7 @@ models:
           - emplois_de_linclusion
           - france_travail
           - fredo
+          - imilo
           - mediation_numerique
           - mes_aides
           - monenfant

pipeline/dbt/models/intermediate/quality/int_quality__stats.sql

@@ -20,6 +20,10 @@
 -- depends_on: {{ ref('int_france_travail__structures') }}
 -- depends_on: {{ ref('stg_fredo__structures') }}
 -- depends_on: {{ ref('int_fredo__structures') }}
+-- depends_on: {{ ref('stg_imilo__offres') }}
+-- depends_on: {{ ref('stg_imilo__structures') }}
+-- depends_on: {{ ref('int_imilo__services') }}
+-- depends_on: {{ ref('int_imilo__structures') }}
 -- depends_on: {{ ref('stg_mediation_numerique__services') }}
 -- depends_on: {{ ref('stg_mediation_numerique__structures') }}
 -- depends_on: {{ ref('int_mediation_numerique__services') }}
@@ -100,15 +104,15 @@ final AS (
     {% for source_node in graph.sources.values() if source_node.source_meta.is_provider %}
 
         {% if source_node.meta.kind %}
+            {% if not loop.first %}
+                UNION ALL
+            {% endif %}
+
 
             {% set source_name = source_node.source_name %}
             {% set stream_name = source_node.name %}
 
             SELECT * FROM {{ source_name }}__{{ stream_name }}__stats
-            {% if not loop.last %}
-                UNION ALL
-            {% endif %}
-
         {% endif %}
 
     {% endfor %}

pipeline/dbt/models/intermediate/sources/imilo/_imilo__models.yml

@@ -0,0 +1,49 @@
+version: 2
+
+models:
+  - name: int_imilo__adresses
+    data_tests:
+      - check_adresse:
+          config:
+            severity: warn
+    columns:
+      - name: id
+        data_tests:
+          - unique
+          - not_null
+
+  - name: int_imilo__services
+    data_tests:
+      - check_service:
+          config:
+            severity: warn
+    columns:
+      - name: id
+        data_tests:
+          - unique
+          - not_null
+          - dbt_utils.not_empty_string
+      - name: structure_id
+        data_tests:
+          - not_null
+          - relationships:
+              to: ref('int_imilo__structures')
+              field: id
+
+  - name: int_imilo__structures
+    data_tests:
+      - check_structure:
+          config:
+            severity: warn
+    columns:
+      - name: id
+        data_tests:
+          - unique
+          - not_null
+          - dbt_utils.not_empty_string
+      - name: adresse_id
+        data_tests:
+          - not_null
+          - relationships:
+              to: ref('int_imilo__adresses')
+              field: id

pipeline/dbt/models/intermediate/sources/imilo/int_imilo__adresses.sql

@@ -0,0 +1,19 @@
+WITH structures AS (
+    SELECT * FROM {{ ref('stg_imilo__structures') }}
+),
+
+final AS (
+    SELECT
+        id                  AS "id",
+        commune             AS "commune",
+        code_postal         AS "code_postal",
+        code_insee          AS "code_insee",
+        adresse             AS "adresse",
+        complement_adresse  AS "complement_adresse",
+        CAST(NULL AS FLOAT) AS "longitude",
+        CAST(NULL AS FLOAT) AS "latitude",
+        _di_source_id       AS "source"
+    FROM structures
+)
+
+SELECT * FROM final

pipeline/dbt/models/intermediate/sources/imilo/int_imilo__services.sql

@@ -0,0 +1,44 @@
+WITH services AS (
+    SELECT * FROM {{ ref('stg_imilo__offres') }}
+),
+
+final AS (
+    SELECT
+        _di_source_id                         AS "source",
+        structure_id                          AS "structure_id",
+        NULL                                  AS "courriel",
+        CAST(NULL AS BOOLEAN)                 AS "cumulable",
+        CAST(NULL AS BOOLEAN)                 AS "contact_public",
+        NULL                                  AS "contact_nom_prenom",
+        CAST(date_maj AS DATE)                AS "date_maj",
+        CAST(date_creation AS DATE)           AS "date_creation",
+        NULL                                  AS "formulaire_en_ligne",
+        NULL                                  AS "frais_autres",
+        CAST(NULL AS TEXT [])                 AS "justificatifs",
+        NULL                                  AS "lien_source",
+        CAST(NULL AS TEXT [])                 AS "modes_accueil",
+        CAST(NULL AS TEXT [])                 AS "modes_orientation_accompagnateur",
+        NULL                                  AS "modes_orientation_accompagnateur_autres",
+        ARRAY[modes_orientation_beneficiaire] AS "modes_orientation_beneficiaire",
+        NULL                                  AS "modes_orientation_beneficiaire_autres",
+        nom                                   AS "nom",
+        NULL                                  AS "page_web",
+        NULL                                  AS "presentation_detail",
+        presentation_resume                   AS "presentation_resume",
+        NULL                                  AS "prise_rdv",
+        ARRAY[profils]                        AS "profils",
+        CAST(NULL AS TEXT [])                 AS "pre_requis",
+        NULL                                  AS "recurrence",
+        ARRAY[thematiques]                    AS "thematiques",
+        CAST(NULL AS TEXT [])                 AS "types",
+        NULL                                  AS "telephone",
+        CAST(NULL AS TEXT [])                 AS "frais",
+        NULL                                  AS "zone_diffusion_type",
+        NULL                                  AS "zone_diffusion_code",
+        NULL                                  AS "zone_diffusion_nom",
+        CAST(NULL AS DATE)                    AS "date_suspension",
+        id                                    AS "id"
+    FROM services
+)
+
+SELECT * FROM final

pipeline/dbt/models/intermediate/sources/imilo/int_imilo__structures.sql

@@ -0,0 +1,31 @@
+WITH structures AS (
+    SELECT * FROM {{ ref('stg_imilo__structures') }}
+),
+
+final AS (
+    SELECT
+        _di_source_id           AS "source",
+        id                      AS "id",
+        NULL                    AS "siret",
+        NULL                    AS "rna",
+        courriel                AS "courriel",
+        antenne                 AS "antenne",
+        horaires_ouverture      AS "horaires_ouverture",
+        site_web                AS "site_web",
+        NULL                    AS "lien_source",
+        NULL                    AS "accessibilite",
+        telephone               AS "telephone",
+        typologie               AS "typologie",
+        nom                     AS "nom",
+        ARRAY[labels_nationaux] AS "labels_nationaux",
+        CAST(NULL AS TEXT [])   AS "labels_autres",
+        presentation_resume     AS "presentation_resume",
+        presentation_detail     AS "presentation_detail",
+        id                      AS "adresse_id",
+        CAST(NULL AS TEXT [])   AS "thematiques",
+        CAST(date_maj AS DATE)  AS "date_maj"
+    FROM structures
+
+)
+
+SELECT * FROM final