Merge pull request #1347 from the-deep/feature/geo-parents

Support: GeoArea Filter + Parent information

apps/ary/tests/test_apis.py

@@ -267,7 +267,7 @@ def test_ary_copy(self):
         # New ARY Count (after assessment-copy)
         for project, _, new_ary_count in ary_stats:
             current_ary_count = Assessment.objects.filter(project_id=project.pk).count()
-            assert new_ary_count == current_ary_count,\
+            assert new_ary_count == current_ary_count, \
                 f'Project: {project.title} {project.pk} Assessment new count is different'
 
     def test_filter_assessment(self):

apps/deep_migration/management/commands/migrate_af_changes_v2_v3.py

@@ -275,7 +275,7 @@ def multiselect_property_convertor(properties):
         ('order', 'order', True),
     ]
     options = (
-        properties if type(properties) == list
+        properties if isinstance(properties, list)
         else properties['options']
     )
     new_options = []
@@ -461,7 +461,7 @@ def geo_attribute_data_convertor(data):
     polygons = []
     points = []
     for value in values:
-        if type(value) is dict:
+        if isinstance(value, dict):
             value_type = value.get('type')
             if value_type == 'Point':
                 points.append(value)

apps/deepl_integration/handlers.py

@@ -71,9 +71,9 @@ def generate_file_url_for_new_deepl_server(file):
 
 
 def custom_error_handler(exception, url=None):
-    if type(exception) == requests.exceptions.ConnectionError:
+    if isinstance(exception, requests.exceptions.ConnectionError):
         raise serializers.ValidationError(f'ConnectionError on provided file: {url}')
-    if type(exception) == json.decoder.JSONDecodeError:
+    if isinstance(exception, json.decoder.JSONDecodeError):
         raise serializers.ValidationError(f'Failed to parse provided json file: {url}')
     raise serializers.ValidationError(f'Failed to handle the provided file: <error={type(exception)}>: {url}')
 

apps/entry/filter_set.py

@@ -236,7 +236,7 @@ def authoring_organization_types_filter(self, qs, name, value):
                     'lead__authors__organization_type'
                 )
             )
-            if type(value[0]) == OrganizationType:
+            if isinstance(value[0], OrganizationType):
                 return qs.filter(organization_types__in=[ot.id for ot in value]).distinct()
             return qs.filter(organization_types__in=value).distinct()
         return qs
@@ -292,7 +292,7 @@ def get_filtered_entries_using_af_filter(
             .annotate(adminlevel_count=models.Count('adminlevel'))\
             .aggregate(max_level=models.Max('adminlevel_count'))['max_level'] or 0
 
-    if type(queries) == list:
+    if isinstance(queries, list):
         queries = {
             q['filter_key']: q
             for q in queries
@@ -651,7 +651,7 @@ def authoring_organization_types_filter(self, qs, name, value):
                     'lead__authors__organization_type'
                 )
             )
-            if type(value[0]) == OrganizationType:
+            if isinstance(value[0], OrganizationType):
                 return qs.filter(organization_types__in=[ot.id for ot in value]).distinct()
             return qs.filter(organization_types__in=value).distinct()
         return qs

apps/entry/serializers.py

@@ -274,7 +274,7 @@ def validate(self, data):
                 lead,
                 request.user,
             )
-            if type(generated_image) == File:
+            if isinstance(generated_image, File):
                 data['image'] = generated_image
         return data
 
@@ -693,7 +693,7 @@ def validate(self, data):
             data['image'] = lead_image.clone_as_deep_file(request.user)
         elif image_raw:
             generated_image = base64_to_deep_image(image_raw, lead, request.user)
-            if type(generated_image) == File:
+            if isinstance(generated_image, File):
                 data['image'] = generated_image
 
         return data

apps/entry/stats.py

@@ -231,7 +231,7 @@ def get_project_entries_stats(project, skip_geo_data=False):
 
     # Make sure this are array
     for key in ['widget1d', 'widget2d']:
-        if type(config[key]) is not list:
+        if not isinstance(config[key], list):
             config[key] = [config[key]]
 
     w_reliability_default = w_severity_default = w_multiselect_widget_default = w_organigram_widget_default = {

apps/export/formats/xlsx.py

@@ -14,7 +14,7 @@
 
 
 def xstr(value):
-    if type(value) == int:
+    if isinstance(value, int):
         return str(value)
     return get_valid_xml_string(value, escape=False)
 

apps/export/models.py

@@ -21,7 +21,7 @@
 def export_upload_to(instance, filename: str) -> str:
     random_string = get_random_string(length=10)
     prefix = 'export'
-    if type(instance) == GenericExport:
+    if isinstance(instance, GenericExport):
         prefix = 'global-export'
     return f'{prefix}/{random_string}/{filename}'
 

apps/geo/dataloaders.py

@@ -5,9 +5,7 @@
 
 from utils.graphene.dataloaders import DataLoaderWithContext, WithContextMixin
 
-from .models import (
-    AdminLevel,
-)
+from .models import AdminLevel
 
 
 class AdminLevelLoader(DataLoaderWithContext):

apps/geo/factories.py

@@ -18,7 +18,7 @@ class Meta:
 
 
 class AdminLevelFactory(DjangoModelFactory):
-    title = factory.Sequence(lambda n: f'Region-{n}')
+    title = factory.Sequence(lambda n: f'Admin-Level-{n}')
 
     class Meta:
         model = AdminLevel

apps/geo/filter_set.py

@@ -91,12 +91,14 @@ class Meta:
 # ------------------------------ Graphql filters -----------------------------------
 class GeoAreaGqlFilterSet(OrderEnumMixin, django_filters.rest_framework.FilterSet):
     ids = IDListFilter(field_name='id')
+    region_ids = IDListFilter(field_name='admin_level__region')
+    admin_level_ids = IDListFilter(field_name='admin_level')
     search = django_filters.CharFilter(
         label='Geo Area Label search',
         method='geo_area_label'
     )
     titles = StringListFilter(
-        label='Geo Area Label search',
+        label='Geo Area Label search (Multiple titles)',
         method='filter_titles'
     )
     ordering = MultipleInputFilter(GeoAreaOrderingEnum, method='ordering_filter')

apps/geo/management/commands/retrigger_geo_cache.py

@@ -0,0 +1,46 @@
+import logging
+from django.core.management.base import BaseCommand
+from django.db import transaction
+
+from geo.models import Region, AdminLevel
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Re-trigger cached data for all geo entities. For specific objects admin panel'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--region',
+            action='store_true',
+            help='Calculate all regions cache',
+        )
+        parser.add_argument(
+            '--admin-level',
+            action='store_true',
+            help='Calculate all regions admin level cache',
+        )
+
+    def calculate(self, Model):
+        success_ids = []
+        qs = Model.objects.all()
+        total = qs.count()
+        for index, item in enumerate(qs.all(), start=1):
+            try:
+                self.stdout.write(f"({index}/{total}) [{item.pk}] {item.title}")
+                with transaction.atomic():
+                    item.calc_cache()
+                success_ids.append(item.pk)
+            except Exception:
+                logger.error(f'{Model.__name__} Cache Calculation Failed!!', exc_info=True)
+        self.stdout.write(self.style.SUCCESS(f'{success_ids=}'))
+
+    def handle(self, *_, **options):
+        calculate_regions = options['region']
+        calculate_admin_levels = options['admin_level']
+
+        if calculate_regions:
+            self.calculate(Region)
+        if calculate_admin_levels:
+            self.calculate(AdminLevel)

apps/geo/migrations/0042_rename_data_geoarea_cached_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.17 on 2023-08-02 09:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('geo', '0041_geoarea_centroid'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='geoarea',
+            old_name='data',
+            new_name='cached_data',
+        ),
+    ]

apps/geo/models.py

@@ -3,6 +3,7 @@
 from typing import List, Union
 from django.contrib.gis.db import models
 from django.core.serializers import serialize
+from django.db import transaction, connection
 from django.contrib.gis.gdal import Envelope
 from django.contrib.gis.db.models.aggregates import Union as PgUnion
 from django.contrib.gis.db.models.functions import Centroid
@@ -191,12 +192,60 @@ def get_geo_area_titles(self):
         return self.geo_area_titles
 
     def calc_cache(self, save=True):
-        # GeoJSON
+        # Update geo parent_titles data
+        with transaction.atomic():
+            GEO_PARENT_DATA_CALC_SQL = f'''
+                WITH geo_parents_data as (
+                  SELECT
+                    id,
+                    title,
+                    (
+                      WITH RECURSIVE parents AS (
+                       SELECT
+                        sub_g.id,
+                        sub_g.title,
+                        sub_g.parent_id,
+                        sub_g.admin_level_id,
+                        sub_adminlevel.level
+                       FROM {GeoArea._meta.db_table} sub_g
+                         INNER JOIN {AdminLevel._meta.db_table} AS sub_adminlevel
+                            ON sub_adminlevel.id = sub_g.admin_level_id
+                       WHERE sub_g.id = outer_g.parent_id
+                       UNION
+                       SELECT
+                        sub_parent_g.id,
+                        sub_parent_g.title,
+                        sub_parent_g.parent_id,
+                        sub_parent_g.admin_level_id,
+                        sub_parent_adminlevel.level
+                       FROM {GeoArea._meta.db_table} AS sub_parent_g
+                         INNER JOIN parents ON sub_parent_g.id = parents.parent_id
+                         INNER JOIN {AdminLevel._meta.db_table} AS sub_parent_adminlevel
+                            ON sub_parent_adminlevel.id = sub_parent_g.admin_level_id
+                      ) SELECT array_agg(title ORDER BY level) from parents
+                    ) as parent_titles
+                  FROM {GeoArea._meta.db_table} as outer_g
+                  WHERE admin_level_id = %(admin_level_id)s
+                )
+                UPDATE {GeoArea._meta.db_table} AS G
+                SET
+                    cached_data = JSONB_SET(
+                        COALESCE(G.cached_data, '{{}}'),
+                        '{{parent_titles}}',
+                        TO_JSONB(GP.parent_titles), true
+                    )
+                FROM geo_parents_data GP
+                WHERE
+                    G.id = GP.id
+            '''
+            with connection.cursor() as cursor:
+                cursor.execute(GEO_PARENT_DATA_CALC_SQL, {'admin_level_id': self.pk})
+
         geojson = json.loads(serialize(
             'geojson',
             self.geoarea_set.all(),
             geometry_field='polygons',
-            fields=('pk', 'title', 'code', 'parent'),
+            fields=('pk', 'title', 'code', 'cached_data'),
         ))
 
         # Titles
@@ -286,18 +335,21 @@ class GeoArea(models.Model):
     An actual geo area in a given admin level
     """
     admin_level = models.ForeignKey(AdminLevel, on_delete=models.CASCADE)
-    parent = models.ForeignKey('GeoArea',
-                               on_delete=models.SET_NULL,
-                               null=True, blank=True, default=None)
+    parent = models.ForeignKey(
+        'GeoArea',
+        on_delete=models.SET_NULL,
+        null=True, blank=True, default=None,
+    )
     title = models.CharField(max_length=255)
     code = models.CharField(max_length=255, blank=True)
-    data = models.JSONField(default=None, blank=True, null=True)
 
     # TODO Rename to geometry
     polygons = models.GeometryField(null=True, blank=True, default=None)
 
     # Cache
     centroid = models.PointField(blank=True, null=True)
+    # -- Used to store additional data
+    cached_data = models.JSONField(default=None, blank=True, null=True)
 
     def __str__(self):
         return self.title
@@ -316,10 +368,11 @@ def sync_centroid(cls):
 
     @classmethod
     def get_for_project(cls, project, is_published=True):
-        return cls.objects.filter(
-            admin_level__region__is_published=is_published,
-            admin_level__region__project=project,
+        admin_levels_qs = AdminLevel.objects.filter(
+            region__is_published=is_published,
+            region__project=project,
         )
+        return cls.objects.filter(admin_level__in=admin_levels_qs)
 
     def clone_to(self, admin_level, parent=None):
         geo_area = GeoArea(
@@ -363,5 +416,4 @@ def can_modify(self, user):
         return self.admin_level.can_modify(user)
 
     def get_label(self):
-        return '{} / {}'.format(self.admin_level.title,
-                                self.title)
+        return '{} / {}'.format(self.admin_level.title, self.title)

apps/geo/schema.py

@@ -20,11 +20,18 @@ def get_users_adminlevel_qs(info):
     return AdminLevel.get_for(info.context.user).defer('geo_area_titles')
 
 
-def get_geo_area_queryset_for_project_geo_area_type(queryset=None):
-    return (queryset or GeoArea.objects).annotate(
+def get_geo_area_queryset_for_project_geo_area_type(queryset=None, defer_fields=('polygons', 'centroid')):
+    _queryset = queryset
+    if _queryset is None:
+        _queryset = GeoArea.objects
+    _queryset = _queryset.annotate(
         region_title=models.F('admin_level__region__title'),
         admin_level_title=models.F('admin_level__title'),
+        admin_level_level=models.F('admin_level__level'),
     )
+    if defer_fields:
+        _queryset = _queryset.defer(*defer_fields)
+    return _queryset
 
 
 class AdminLevelType(DjangoObjectType):
@@ -106,6 +113,11 @@ class Meta:
 
     region_title = graphene.String(required=True)
     admin_level_title = graphene.String(required=True)
+    admin_level_level = graphene.Int()
+    parent_titles = graphene.List(graphene.NonNull(graphene.String), required=True)
+
+    def resolve_parent_titles(root, info, **kwargs):
+        return (root.cached_data or {}).get('parent_titles') or []
 
 
 class ProjectGeoAreaListType(CustomDjangoListObjectType):