metabase: Add gps data

clevercloud/crons/populate_metabase_emplois.sh

@@ -39,6 +39,8 @@ if [[ "$1" == "--daily" ]]; then
     django-admin populate_metabase_emplois --mode=users |& tee -a "$OUTPUT_LOG"
     django-admin populate_metabase_emplois --mode=memberships |& tee -a "$OUTPUT_LOG"
     django-admin populate_metabase_emplois --mode=dbt_daily |& tee -a "$OUTPUT_LOG"
+    django-admin populate_metabase_emplois --mode=gps_groups |& tee -a "$OUTPUT_LOG"
+    django-admin populate_metabase_emplois --mode=gps_memberships |& tee -a "$OUTPUT_LOG"
     django-admin populate_metabase_emplois --mode=data_inconsistencies |& tee -a "$OUTPUT_LOG"
     django-admin send_slack_message ":white_check_mark: succès mise à jour de données C1 -> Metabase"
 elif [[ "$1" == "--monthly" ]]; then

itou/metabase/management/commands/populate_metabase_emplois.py

@@ -36,6 +36,7 @@
 from itou.companies.models import Company, CompanyMembership, JobDescription
 from itou.eligibility.enums import AdministrativeCriteriaLevel
 from itou.eligibility.models import AdministrativeCriteria, EligibilityDiagnosis
+from itou.gps.models import FollowUpGroup, FollowUpGroupMembership
 from itou.institutions.models import Institution, InstitutionMembership
 from itou.job_applications.enums import JobApplicationState, Origin, RefusalReason, SenderKind
 from itou.job_applications.models import JobApplication
@@ -51,6 +52,7 @@
     evaluated_job_applications,
     evaluated_siaes,
     evaluation_campaigns,
+    gps,
     insee_codes,
     institutions,
     job_applications,
@@ -113,6 +115,8 @@ def __init__(self, *args, **kwargs):
             "departments": self.populate_departments,
             "enums": self.populate_enums,
             "dbt_daily": self.build_dbt_daily,
+            "gps_groups": self.populate_gps_groups,
+            "gps_memberships": self.populate_gps_memberships,
             "data_inconsistencies": self.report_data_inconsistencies,
         }
 
@@ -495,6 +499,32 @@ def populate_enums(self):
             df = get_df_from_rows(rows)
             store_df(df=df, table_name=table_name)
 
+    def populate_gps_groups(self):
+        queryset = FollowUpGroup.objects.all().annotate(beneficiary_department=F("beneficiary__department"))
+        populate_table(gps.GroupsTable, batch_size=100_000, querysets=[queryset])
+
+    def populate_gps_memberships(self):
+        queryset = (
+            FollowUpGroupMembership.objects.all()
+            .annotate(
+                companies_departments=ArrayAgg(
+                    "member__companymembership__company__department",
+                    filter=Q(member__companymembership__is_active=True),
+                    distinct=True,
+                    ordering="member__companymembership__company__department",
+                )
+            )
+            .annotate(
+                prescriber_departments=ArrayAgg(
+                    "member__prescribermembership__organization__department",
+                    filter=Q(member__prescribermembership__is_active=True),
+                    distinct=True,
+                    ordering="member__prescribermembership__organization__department",
+                )
+            )
+        )
+        populate_table(gps.MembershipsTable, batch_size=100_000, querysets=[queryset])
+
     @timeit
     def report_data_inconsistencies(self):
         """

itou/metabase/tables/gps.py

@@ -0,0 +1,48 @@
+from itou.gps.models import FollowUpGroup, FollowUpGroupMembership
+from itou.metabase.tables.utils import MetabaseTable, get_column_from_field
+
+
+def get_group_field(name):
+    return FollowUpGroup._meta.get_field(name)
+
+
+GroupsTable = MetabaseTable(name="gps_groups_v1")
+GroupsTable.add_columns(
+    [
+        get_column_from_field(get_group_field("id"), "id"),
+        get_column_from_field(get_group_field("created_at"), "created_at"),
+        get_column_from_field(get_group_field("updated_at"), "updated_at"),
+        get_column_from_field(get_group_field("created_in_bulk"), "created_in_bulk"),
+        {
+            "name": "department",
+            "type": "text",
+            "comment": "Département du bénéficiaire",
+            "fn": lambda o: o.beneficiary_department,
+        },
+    ]
+)
+
+
+def get_membership_field(name):
+    return FollowUpGroupMembership._meta.get_field(name)
+
+
+MembershipsTable = MetabaseTable(name="gps_membres_v1")
+MembershipsTable.add_columns(
+    [
+        get_column_from_field(get_membership_field("id"), "id"),
+        get_column_from_field(get_membership_field("follow_up_group_id"), "group_id"),
+        get_column_from_field(get_membership_field("created_at"), "created_at"),
+        get_column_from_field(get_membership_field("updated_at"), "updated_at"),
+        get_column_from_field(get_membership_field("ended_at"), "ended_at"),
+        get_column_from_field(get_membership_field("is_referent"), "is_referent"),
+        get_column_from_field(get_membership_field("member_id"), "member_id"),
+        {
+            "name": "org_departments",
+            "type": "text[]",
+            "comment": "Départements de l'organisation",
+            "fn": lambda o: o.companies_departments or o.prescriber_departments,
+        },
+        get_column_from_field(get_membership_field("created_in_bulk"), "created_in_bulk"),
+    ]
+)

tests/metabase/management/test_populate_metabase_emplois.py

@@ -15,6 +15,7 @@
 from itou.eligibility.models import AdministrativeCriteria
 from itou.geo.utils import coords_to_geometry
 from itou.job_applications.enums import JobApplicationState
+from itou.metabase.tables import gps
 from itou.metabase.tables.utils import hash_content
 from itou.users.enums import IdentityProvider, UserKind
 from tests.analytics.factories import DatumFactory, StatsDashboardVisitFactory
@@ -27,6 +28,7 @@
 from tests.companies.factories import CompanyFactory, CompanyMembershipFactory, JobDescriptionFactory
 from tests.eligibility.factories import IAEEligibilityDiagnosisFactory
 from tests.geo.factories import QPVFactory
+from tests.gps.factories import FollowUpGroupFactory, FollowUpGroupMembershipFactory
 from tests.institutions.factories import InstitutionFactory, InstitutionMembershipFactory
 from tests.job_applications.factories import JobApplicationFactory
 from tests.jobs.factories import create_test_romes_and_appellations
@@ -1207,3 +1209,81 @@ def test_populate_companies():
                 datetime.date(2023, 2, 1),
             ),
         ]
+
+
+@freeze_time("2023-02-02")
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.usefixtures("metabase")
+def test_populate_gps_groups():
+    group = FollowUpGroupFactory(for_snapshot=True)
+
+    num_queries = 1  # Count FollowUpGroups
+    num_queries += 1  # COMMIT Queryset counts (autocommit mode)
+    num_queries += 1  # COMMIT Create table
+    num_queries += 1  # Select FollowUpGroups pks
+    num_queries += 1  # Select one chunk of FollowUpGroups
+    num_queries += 1  # Select FollowUpGroups with annotations
+    num_queries += 1  # COMMIT (inject_chunk)
+    num_queries += 1  # COMMIT (rename_table_atomically DROP TABLE)
+    num_queries += 1  # COMMIT (rename_table_atomically RENAME TABLE)
+    num_queries += 1  # COMMIT (rename_table_atomically DROP TABLE)
+    with assertNumQueries(num_queries):
+        management.call_command("populate_metabase_emplois", mode="gps_groups")
+
+    with connection.cursor() as cursor:
+        cursor.execute(f"SELECT * FROM {gps.GroupsTable.name} ORDER BY id")
+        rows = cursor.fetchall()
+        assert len(rows) == 1
+        assert rows == [
+            (
+                group.pk,
+                group.created_at,
+                group.updated_at,
+                group.created_in_bulk,
+                group.beneficiary.department,
+                datetime.date(2023, 2, 1),
+            ),
+        ]
+
+
+@freeze_time("2023-02-02")
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.usefixtures("metabase")
+def test_populate_gps_memberships():
+    membership = FollowUpGroupMembershipFactory(follow_up_group__for_snapshot=True, member__for_snapshot=True)
+    prescriber = membership.member
+    PrescriberMembershipFactory(user=prescriber, organization__department="63")
+    PrescriberMembershipFactory(user=prescriber, organization__department="13")
+    PrescriberMembershipFactory(user=prescriber, organization__department="75")
+
+    num_queries = 1  # Count FollowUpGroupMemberships
+    num_queries += 1  # COMMIT Queryset counts (autocommit mode)
+    num_queries += 1  # COMMIT Create table
+    num_queries += 1  # Select FollowUpGroupMemberships pks
+    num_queries += 1  # Select one chunk of FollowUpGroupMemberships
+    num_queries += 1  # Select FollowUpGroupMemberships with annotations
+    num_queries += 1  # COMMIT (inject_chunk)
+    num_queries += 1  # COMMIT (rename_table_atomically DROP TABLE)
+    num_queries += 1  # COMMIT (rename_table_atomically RENAME TABLE)
+    num_queries += 1  # COMMIT (rename_table_atomically DROP TABLE)
+    with assertNumQueries(num_queries):
+        management.call_command("populate_metabase_emplois", mode="gps_memberships")
+
+    with connection.cursor() as cursor:
+        cursor.execute(f"SELECT * FROM {gps.MembershipsTable.name} ORDER BY id")
+        rows = cursor.fetchall()
+        assert len(rows) == 1
+        assert rows == [
+            (
+                membership.pk,
+                membership.follow_up_group.pk,
+                membership.created_at,
+                membership.updated_at,
+                membership.ended_at,
+                int(membership.is_referent),
+                membership.member.pk,
+                ["13", "63", "75"],
+                int(membership.created_in_bulk),
+                datetime.date(2023, 2, 1),
+            ),
+        ]