DEDUP ENGINE - All Individuals should have RDI

backend/hct_mis_api/apps/core/celery_tasks.py

@@ -171,6 +171,7 @@ def create_target_population_task(self: Any, storage_id: str, program_id: str, t
                         "relationship": HEAD,
                         "rdi_merge_status": MergeStatusModel.MERGED,
                         "flex_fields": populate_pdu_with_null_values(program),
+                        "registration_data_import": registration_data_import,
                     }
                     if family_id in families:
                         individual = Individual(**individual_data, household_id=families.get(family_id))

backend/hct_mis_api/apps/core/migrations/0084_migration.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-08-16 11:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0083_migration'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='periodicfielddata',
+            name='subtype',
+            field=models.CharField(choices=[('DATE', 'Date'), ('DECIMAL', 'Number'), ('STRING', 'Text'), ('BOOLEAN', 'Boolean (true/false)')], max_length=16),
+        ),
+    ]

backend/hct_mis_api/apps/core/tests/test_edopomoga_tp_creation.py

@@ -23,22 +23,16 @@ class TestEdopomogaCreation(APITestCase):
 
     @classmethod
     def setUpTestData(cls) -> None:
-        create_afghanistan()
+        cls.business_area = create_afghanistan()
         call_command("loadcountries")
         cls.generate_document_types_for_all_countries()
         cls.user = UserFactory.create()
-        cls.business_area = BusinessArea.objects.get(slug="afghanistan")
-
-        country = geo_models.Country.objects.get(name="Afghanistan")
-        cls.business_area.countries.add(country)
-
+        cls.business_area.countries.add(geo_models.Country.objects.get(name="Afghanistan"))
         cls.program = ProgramFactory(
             name="Test program ONE",
             business_area=BusinessArea.objects.first(),
         )
-
         content = Path(f"{settings.PROJECT_ROOT}/apps/core/tests/test_files/edopomoga_sample.csv")
-
         cls.storage_file = StorageFile.objects.create(
             created_by=cls.user,
             business_area=cls.business_area,

backend/hct_mis_api/apps/grievance/services/data_change/add_individual_service.py

@@ -120,6 +120,7 @@ def close(self, user: AbstractUser) -> None:
             business_area=self.grievance_ticket.business_area,
             program_id=household.program_id,
             rdi_merge_status=MergeStatusModel.MERGED,
+            registration_data_import=household.registration_data_import,
             **individual_data,
         )
         individual.refresh_from_db()

backend/hct_mis_api/apps/household/admin/household.py

@@ -446,7 +446,9 @@ def mass_enroll_to_another_program(self, request: HttpRequest, qs: QuerySet) ->
                 program_for_enroll = form.cleaned_data["program_for_enroll"]
                 households_ids = list(qs.distinct("unicef_id").values_list("id", flat=True))
                 enroll_households_to_program_task.delay(
-                    households_ids=households_ids, program_for_enroll_id=str(program_for_enroll.id)
+                    households_ids=households_ids,
+                    program_for_enroll_id=str(program_for_enroll.id),
+                    user_id=str(request.user.id),
                 )
                 self.message_user(
                     request,

backend/hct_mis_api/apps/household/celery_tasks.py

@@ -201,10 +201,10 @@ def revalidate_phone_number_task(individual_ids: List[UUID]) -> None:
 @app.task()
 @log_start_and_end
 @sentry_tags
-def enroll_households_to_program_task(households_ids: List, program_for_enroll_id: str) -> None:
+def enroll_households_to_program_task(households_ids: List, program_for_enroll_id: str, user_id: str) -> None:
     households = Household.objects.filter(pk__in=households_ids)
     program_for_enroll = Program.objects.get(id=program_for_enroll_id)
-    enroll_households_to_program(households, program_for_enroll)
+    enroll_households_to_program(households, program_for_enroll, user_id)
 
 
 @app.task()

backend/hct_mis_api/apps/household/migrations/0185_migration.py

@@ -0,0 +1,50 @@
+# Generated by Django 3.2.25 on 2024-08-16 11:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+from django.db.models import Count
+
+
+def assign_individual_to_rdi(apps, schema_editor):
+    RegistrationDataImport = apps.get_model("registration_data", "RegistrationDataImport")
+    Individual = apps.get_model("household", "Individual")
+    Program = apps.get_model("program", "Program")
+
+    for program in Program.objects.all():
+        # create RDI only if exists any individual without RDI
+        individual_qs = Individual.objects.filter(program=program, registration_data_import__isnull=True)
+        aggregated_data = individual_qs.aggregate(
+            individual_count=Count("id"),
+            household_count=Count("household", distinct=True)
+        )
+        if aggregated_data["individual_count"] > 0:
+            rdi = RegistrationDataImport.objects.create(
+                name=f"RDI for Individuals [data migration for Programme: {program.name}]",
+                status="MERGED",
+                imported_by=None,
+                data_source="XLS",
+                number_of_individuals=aggregated_data["individual_count"],
+                number_of_households=aggregated_data["household_count"],
+                business_area=program.business_area,
+                program_id=program.id,
+                import_data=None,
+            )
+
+            individual_qs.update(registration_data_import_id=rdi.id)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('registration_data', '0038_migration'),
+        ('household', '0184_migration'),
+    ]
+
+    operations = [
+        migrations.RunPython(assign_individual_to_rdi, migrations.RunPython.noop,),
+        migrations.AlterField(
+            model_name='individual',
+            name='registration_data_import',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='individuals', to='registration_data.registrationdataimport'),
+        ),
+    ]

backend/hct_mis_api/apps/household/models.py

@@ -977,7 +977,6 @@ class Individual(
         "registration_data.RegistrationDataImport",
         related_name="individuals",
         on_delete=models.CASCADE,
-        null=True,
     )
     work_status = models.CharField(
         max_length=20,

backend/hct_mis_api/apps/household/tests/test_assign_individual_to_rdi.py

@@ -0,0 +1,114 @@
+from django.core.management import call_command
+from django.db.models import Count
+from django.test import TransactionTestCase
+
+from hct_mis_api.apps.core.fixtures import create_afghanistan
+from hct_mis_api.apps.household.fixtures import create_household_and_individuals
+from hct_mis_api.apps.household.models import Household, Individual
+from hct_mis_api.apps.program.fixtures import ProgramFactory
+from hct_mis_api.apps.program.models import Program
+from hct_mis_api.apps.registration_data.models import RegistrationDataImport
+
+
+class TestMigrationIndAssignToRDI(TransactionTestCase):
+    def test_assign_individual_to_rdi_migration(self) -> None:
+        call_command("migrate", "household", "0184_migration", verbosity=0)
+
+        ba_afghanistan = create_afghanistan()
+        program_1 = ProgramFactory(name="program_1", business_area=ba_afghanistan)
+        program_2 = ProgramFactory(name="program_2", business_area=ba_afghanistan)
+        create_household_and_individuals(
+            household_data={
+                "business_area": ba_afghanistan,
+                "program": program_1,
+            },
+            individuals_data=[
+                {
+                    "business_area": ba_afghanistan,
+                    "program": program_1,
+                },
+            ],
+        )
+        create_household_and_individuals(
+            household_data={
+                "business_area": ba_afghanistan,
+                "program": program_2,
+            },
+            individuals_data=[
+                {
+                    "business_area": ba_afghanistan,
+                    "program": program_2,
+                },
+            ],
+        )
+        # set registration_data_import to null
+        Household.objects.all().update(registration_data_import=None)
+        Individual.objects.all().update(registration_data_import=None)
+        # add more individuals assigned to RDI
+        program_3 = ProgramFactory(name="Program 333", business_area=ba_afghanistan)
+        create_household_and_individuals(
+            household_data={
+                "business_area": ba_afghanistan,
+                "program": program_3,
+            },
+            individuals_data=[
+                {
+                    "business_area": ba_afghanistan,
+                    "program": program_3,
+                },
+                {
+                    "business_area": ba_afghanistan,
+                    "program": program_3,
+                },
+            ],
+        )
+        # check before migrate data
+        self.assertEqual(RegistrationDataImport.objects.count(), 3)
+        self.assertEqual(Program.objects.count(), 3)
+        self.assertEqual(Individual.objects.count(), 4)
+        self.assertEqual(Household.objects.count(), 3)
+        self.assertEqual(Individual.objects.filter(program=program_1).count(), 1)
+        self.assertEqual(Individual.objects.filter(program=program_2).count(), 1)
+        self.assertEqual(Individual.objects.filter(registration_data_import__isnull=True).count(), 2)
+        self.assertEqual(Household.objects.filter(registration_data_import__isnull=True).count(), 2)
+
+        # call_command("migrate", "household", "0185_migration", verbosity=0)
+        for program in Program.objects.all():
+            # create RDI only if exists any individual without RDI
+            individual_qs = Individual.objects.filter(program=program, registration_data_import__isnull=True)
+            aggregated_data = individual_qs.aggregate(
+                individual_count=Count("id"), household_count=Count("household", distinct=True)
+            )
+            if aggregated_data["individual_count"] > 0:
+                rdi = RegistrationDataImport.objects.create(
+                    name=f"RDI for Individuals [data migration for Programme: {program.name}]",
+                    status="MERGED",
+                    imported_by=None,
+                    data_source="XLS",
+                    number_of_individuals=aggregated_data["individual_count"],
+                    number_of_households=aggregated_data["household_count"],
+                    business_area=program.business_area,
+                    program_id=program.id,
+                    import_data=None,
+                )
+
+                individual_qs.update(registration_data_import_id=rdi.id)
+
+        # 3 old RDIs and + 2 new created
+        self.assertEqual(RegistrationDataImport.objects.count(), 5)
+        self.assertEqual(Individual.objects.filter(registration_data_import__isnull=False).count(), 4)
+        self.assertEqual(Household.objects.filter(registration_data_import__isnull=True).count(), 2)
+
+        rdi_1 = RegistrationDataImport.objects.get(
+            name=f"RDI for Individuals [data migration for Programme: {program_1.name}]"
+        )
+        self.assertEqual(rdi_1.status, "MERGED")
+        self.assertEqual(rdi_1.number_of_individuals, 1)
+        self.assertEqual(rdi_1.number_of_households, 1)
+
+        rdi_2 = RegistrationDataImport.objects.get(
+            name=f"RDI for Individuals [data migration for Programme: {program_2.name}]"
+        )
+        self.assertEqual(rdi_2.status, "MERGED")
+        self.assertEqual(rdi_2.number_of_individuals, 1)
+        self.assertEqual(rdi_2.number_of_households, 1)

backend/hct_mis_api/apps/household/tests/test_forms.py

@@ -0,0 +1,29 @@
+from django.test import TestCase
+
+from hct_mis_api.apps.core.fixtures import DataCollectingTypeFactory, create_afghanistan
+from hct_mis_api.apps.household.fixtures import HouseholdFactory, IndividualFactory
+from hct_mis_api.apps.household.forms import MassEnrollForm
+from hct_mis_api.apps.program.fixtures import ProgramFactory
+from hct_mis_api.apps.program.models import Program
+
+
+class MassEnrollFormTest(TestCase):
+    def setUp(self) -> None:
+        afg = create_afghanistan()
+        self.business_area_id = str(afg.id)
+        partial = DataCollectingTypeFactory(
+            business_areas=[afg],
+        )
+        partial.compatible_types.add(partial)
+        self.program = ProgramFactory(
+            name="Test Program 333", business_area_id=afg.id, status=Program.ACTIVE, data_collecting_type=partial
+        )
+        self.household = HouseholdFactory(
+            program=self.program,
+            head_of_household=IndividualFactory(household=None),
+        )
+
+    def test_clean_form(self) -> None:
+        form_data = {"program_for_enroll": self.program.id, "apply": True}
+        form = MassEnrollForm(data=form_data, business_area_id=self.business_area_id, households=[self.household])
+        self.assertTrue(form.is_valid())

backend/hct_mis_api/apps/payment/fixtures.py

@@ -904,6 +904,7 @@ def generate_payment_plan() -> None:
         full_name="Jan Kowalski",
         sex=MALE,
         program=program,
+        registration_data_import=rdi,
         defaults={"individual_collection": IndividualCollectionFactory()},
     )[0]
 
@@ -917,6 +918,7 @@ def generate_payment_plan() -> None:
         full_name="Adam Nowak",
         sex=MALE,
         program=program,
+        registration_data_import=rdi,
         defaults={"individual_collection": IndividualCollectionFactory()},
     )[0]
 

backend/hct_mis_api/apps/payment/migrations/0143_migration.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.25 on 2024-08-22 09:53
+
+from django.db import migrations, models
+import hct_mis_api.apps.payment.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('payment', '0142_migration'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='financialserviceproviderxlsxtemplate',
+            name='flex_fields',
+            field=hct_mis_api.apps.payment.models.FlexFieldArrayField(base_field=models.CharField(blank=True, max_length=255), blank=True, default=list, size=None),
+        ),
+    ]

backend/hct_mis_api/apps/program/celery_tasks.py

@@ -12,10 +12,10 @@
 @app.task()
 @sentry_tags
 @log_start_and_end
-def copy_program_task(copy_from_program_id: str, new_program_id: str) -> None:
+def copy_program_task(copy_from_program_id: str, new_program_id: str, user_id: str) -> None:
     program = Program.objects.get(id=new_program_id)
     set_sentry_business_area_tag(program.business_area.name)
-    copy_program_related_data(copy_from_program_id, program)
+    copy_program_related_data(copy_from_program_id, program, user_id)
     program_copied.send(sender=Program, instance=program)
 
 

backend/hct_mis_api/apps/program/mutations.py

@@ -283,7 +283,9 @@ def processed_mutate(cls, root: Any, info: Any, program_data: Dict) -> "CopyProg
             create_program_partner_access(partners_data, program, partner_access)
 
         transaction.on_commit(
-            lambda: copy_program_task.delay(copy_from_program_id=program_id, new_program_id=program.id)
+            lambda: copy_program_task.delay(
+                copy_from_program_id=program_id, new_program_id=program.id, user_id=str(info.context.user.id)
+            )
         )
 
         if pdu_fields is not None: