chore: upgrade Django 3.2 (#962)

* chore: upgrade Django 3.2

* docs: lint markdown files

* chore: entrypoint scripts

* fix(producer): remove warning

.github/ISSUE_TEMPLATE.md

@@ -1,9 +1,11 @@
-### What version of Aether are you using?
+# Pull request description
 
-### What operating system and processor architecture are you using?
+## What version of Aether are you using?
 
-### What did you do?
+## What operating system and processor architecture are you using?
 
-### What did you expect to see?
+## What did you do?
 
-### What did you see instead?
+## What did you expect to see?
+
+## What did you see instead?

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Introduction
 
-## You want to help out with Aether? Yay!
+## You want to help out with Aether? Yay! :D
 
 First off, thank you for considering contributing to Aether. It's people like you that make Aether *the* platform for data-driven humanitarian projects.
 

aether-kernel/aether/kernel/api/migrations/0112_json_field_2.py

@@ -2,7 +2,7 @@
 
 import aether.kernel.api.validators
 import aether.sdk.drf.fields
-from django.db import migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -15,41 +15,41 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='entity',
             name='payload',
-            field=aether.sdk.drf.fields.AetherJSONField(verbose_name='payload'),
+            field=models.JSONField(verbose_name='payload'),
         ),
         migrations.AlterField(
             model_name='exporttask',
             name='settings',
-            field=aether.sdk.drf.fields.AetherJSONField(default=dict, editable=False, verbose_name='settings'),
+            field=models.JSONField(default=dict, editable=False, verbose_name='settings'),
         ),
         migrations.AlterField(
             model_name='mapping',
             name='definition',
-            field=aether.sdk.drf.fields.AetherJSONField(validators=[aether.kernel.api.validators.wrapper_validate_mapping_definition], verbose_name='mapping rules'),
+            field=models.JSONField(validators=[aether.kernel.api.validators.wrapper_validate_mapping_definition], verbose_name='mapping rules'),
         ),
         migrations.AlterField(
             model_name='mappingset',
             name='input',
-            field=aether.sdk.drf.fields.AetherJSONField(blank=True, null=True, verbose_name='input sample'),
+            field=models.JSONField(blank=True, null=True, verbose_name='input sample'),
         ),
         migrations.AlterField(
             model_name='mappingset',
             name='schema',
-            field=aether.sdk.drf.fields.AetherJSONField(blank=True, null=True, validators=[aether.kernel.api.validators.wrapper_validate_schema_input_definition], verbose_name='AVRO schema'),
+            field=models.JSONField(blank=True, null=True, validators=[aether.kernel.api.validators.wrapper_validate_schema_input_definition], verbose_name='AVRO schema'),
         ),
         migrations.AlterField(
             model_name='schema',
             name='definition',
-            field=aether.sdk.drf.fields.AetherJSONField(validators=[aether.kernel.api.validators.wrapper_validate_schema_definition], verbose_name='AVRO schema'),
+            field=models.JSONField(validators=[aether.kernel.api.validators.wrapper_validate_schema_definition], verbose_name='AVRO schema'),
         ),
         migrations.AlterField(
             model_name='schemadecorator',
             name='topic',
-            field=aether.sdk.drf.fields.AetherJSONField(blank=True, null=True, verbose_name='topic'),
+            field=models.JSONField(blank=True, null=True, verbose_name='topic'),
         ),
         migrations.AlterField(
             model_name='submission',
             name='payload',
-            field=aether.sdk.drf.fields.AetherJSONField(verbose_name='payload'),
+            field=models.JSONField(verbose_name='payload'),
         ),
     ]

aether-kernel/aether/kernel/api/models.py

@@ -32,7 +32,6 @@
 
 from model_utils.models import TimeStampedModel
 
-from aether.sdk.drf.fields import AetherJSONField
 from aether.sdk.multitenancy.models import MtModelAbstract, MtModelChildAbstract
 from aether.sdk.utils import json_prettified, get_file_content
 
@@ -180,13 +179,13 @@ class MappingSet(ExportModelOperationsMixin('kernel_mappingset'), ProjectChildAb
 
     '''
 
-    schema = AetherJSONField(
+    schema = models.JSONField(
         null=True,
         blank=True,
         validators=[wrapper_validate_schema_input_definition],
         verbose_name=_('AVRO schema'),
     )
-    input = AetherJSONField(null=True, blank=True, verbose_name=_('input sample'))
+    input = models.JSONField(null=True, blank=True, verbose_name=_('input sample'))
 
     project = models.ForeignKey(to=Project, on_delete=models.CASCADE, verbose_name=_('project'))
 
@@ -250,7 +249,7 @@ class Submission(ExportModelOperationsMixin('kernel_submission'), ProjectChildAb
 
     '''
 
-    payload = AetherJSONField(verbose_name=_('payload'))
+    payload = models.JSONField(verbose_name=_('payload'))
 
     mappingset = models.ForeignKey(
         to=MappingSet,
@@ -400,7 +399,7 @@ class Schema(ExportModelOperationsMixin('kernel_schema'), KernelAbstract):
 
     name = models.TextField(unique=True, verbose_name=_('name'))
     type = models.TextField(default=NAMESPACE, verbose_name=_('schema type'))
-    definition = AetherJSONField(validators=[wrapper_validate_schema_definition], verbose_name=_('AVRO schema'))
+    definition = models.JSONField(validators=[wrapper_validate_schema_definition], verbose_name=_('AVRO schema'))
 
     # this field is used to group different schemas created automatically
     # from different sources but that share a common structure
@@ -455,7 +454,7 @@ class SchemaDecorator(ExportModelOperationsMixin('kernel_schemadecorator'), Proj
     transport_rule = models.TextField(null=True, blank=True, verbose_name=_('transport rule'))
     masked_fields = models.TextField(null=True, blank=True, verbose_name=_('masked fields'))
     is_encrypted = models.BooleanField(default=False, verbose_name=_('encrypted?'))
-    topic = AetherJSONField(null=True, blank=True, verbose_name=_('topic'))
+    topic = models.JSONField(null=True, blank=True, verbose_name=_('topic'))
 
     project = models.ForeignKey(to=Project, on_delete=models.CASCADE, verbose_name=_('project'))
     schema = models.ForeignKey(to=Schema, on_delete=models.CASCADE, verbose_name=_('schema'))
@@ -507,7 +506,7 @@ class Mapping(ExportModelOperationsMixin('kernel_mapping'), ProjectChildAbstract
 
     '''
 
-    definition = AetherJSONField(
+    definition = models.JSONField(
         validators=[wrapper_validate_mapping_definition],
         verbose_name=_('mapping rules'),
     )
@@ -599,7 +598,7 @@ class Entity(ExportModelOperationsMixin('kernel_entity'), ProjectChildAbstract):
 
     modified = models.CharField(max_length=100, editable=False, verbose_name=_('modified'))
 
-    payload = AetherJSONField(verbose_name=_('payload'))
+    payload = models.JSONField(verbose_name=_('payload'))
     status = models.CharField(max_length=20, choices=ENTITY_STATUS_CHOICES, verbose_name=_('status'))
 
     submission = models.ForeignKey(
@@ -769,7 +768,7 @@ class ExportTask(ExportModelOperationsMixin('kernel_exporttask'), ProjectChildAb
         to=get_user_model(),
         verbose_name=_('Requested by'),
     )
-    settings = AetherJSONField(
+    settings = models.JSONField(
         default=dict,
         editable=False,
         verbose_name=_('settings'),

aether-kernel/conf/pip/primary-requirements.txt

@@ -17,7 +17,7 @@
 
 # Aether Django SDK library with extras
 aether.sdk[cache,server,storage,test]
-django<3.2
+
 
 # Aether Python library
 aether.python

aether-kernel/conf/pip/requirements.txt

@@ -13,7 +13,7 @@
 ################################################################################
 
 aether.python==1.3.0
-aether.sdk==1.3.8
+aether.sdk==1.4.0
 asgiref==3.4.1
 attrs==21.2.0
 autopep8==1.5.7
@@ -29,7 +29,7 @@ coreapi==2.3.3
 coreschema==0.0.4
 coverage==5.5
 decorator==5.1.0
-Django==3.1.13
+Django==3.2.7
 django-cacheops==6.0
 django-cleanup==5.2.0
 django-cors-headers==3.8.0

aether-kernel/entrypoint.sh

@@ -70,8 +70,12 @@ function backup_db {
     if psql -c "" $DB_NAME; then
         echo "$DB_NAME database exists!"
 
-        pg_dump $DB_NAME > ${BACKUPS_FOLDER}/${DB_NAME}-backup-$(date "+%Y%m%d%H%M%S").sql
-        echo "$DB_NAME database backup created."
+        mkdir -p $BACKUPS_FOLDER
+        local BACKUP_FILE=$BACKUPS_FOLDER/$DB_NAME-backup-$(date "+%Y%m%d%H%M%S").sql
+
+        pg_dump $DB_NAME > $BACKUP_FILE
+        chown -f aether:aether $BACKUP_FILE
+        echo "$DB_NAME database backup created in [$BACKUP_FILE]."
     fi
 }
 
@@ -136,6 +140,7 @@ function setup {
     cp /var/tmp/REVISION ${STATIC_DIR}/REVISION 2>/dev/null || true
 
     ./manage.py collectstatic --noinput --verbosity 0
+    chown -Rf aether:aether ${STATIC_ROOT}
     chmod -R 755 ${STATIC_ROOT}
 }
 
@@ -188,6 +193,8 @@ case "$1" in
 
     manage )
         ./manage.py "${@:2}"
+        # required to change migration files owner
+        chown -Rf aether:aether *
     ;;
 
     pip_freeze )
@@ -232,7 +239,7 @@ case "$1" in
     start )
         # ensure that DEBUG mode is disabled
         export DEBUG=
-        # Export woraround: in seconds: 20min
+        # Export workaround: in seconds: 20min
         export UWSGI_HARAKIRI=${UWSGI_HARAKIRI:-1200}
 
         setup

aether-odk-module/aether/odk/api/migrations/0106_json_field_2.py

@@ -1,7 +1,7 @@
 # Generated by Django 3.1.11 on 2021-05-27 12:57
 
 import aether.sdk.drf.fields
-from django.db import migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -14,6 +14,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='xform',
             name='avro_schema',
-            field=aether.sdk.drf.fields.AetherJSONField(blank=True, editable=False, null=True, verbose_name='AVRO schema'),
+            field=models.JSONField(blank=True, editable=False, null=True, verbose_name='AVRO schema'),
         ),
     ]

aether-odk-module/aether/odk/api/models.py

@@ -30,7 +30,6 @@
 
 from django_prometheus.models import ExportModelOperationsMixin
 
-from aether.sdk.drf.fields import AetherJSONField
 from aether.sdk.multitenancy.models import MtModelAbstract, MtModelChildAbstract
 from aether.sdk.utils import json_prettified, get_file_content
 
@@ -181,7 +180,7 @@ class XForm(ExportModelOperationsMixin('odk_xform'), MtModelChildAbstract):
     form_id = models.TextField(default='', editable=False, verbose_name=_('xForm ID'))
     version = models.TextField(default='0', blank=True, verbose_name=_('xForm version'))
     md5sum = models.CharField(default='', editable=False, max_length=36, verbose_name=_('xForm md5sum'))
-    avro_schema = AetherJSONField(null=True, blank=True, editable=False, verbose_name=_('AVRO schema'))
+    avro_schema = models.JSONField(null=True, blank=True, editable=False, verbose_name=_('AVRO schema'))
 
     # This is needed to submit data to kernel
     kernel_id = models.UUIDField(

aether-odk-module/conf/pip/primary-requirements.txt

@@ -17,7 +17,6 @@
 
 # Aether Django SDK library with extras
 aether.sdk[cache,server,storage,test]
-django<3.2
 
 
 # xForm and data manipulation

aether-odk-module/conf/pip/requirements.txt

@@ -12,7 +12,7 @@
 #
 ################################################################################
 
-aether.sdk==1.3.8
+aether.sdk==1.4.0
 asgiref==3.4.1
 autopep8==1.5.7
 boto3==1.18.40
@@ -23,7 +23,7 @@ cffi==1.14.6
 charset-normalizer==2.0.4
 configparser==5.0.2
 coverage==5.5
-Django==3.1.13
+Django==3.2.7
 django-cacheops==6.0
 django-cleanup==5.2.0
 django-cors-headers==3.8.0

aether-odk-module/entrypoint.sh

@@ -71,8 +71,12 @@ function backup_db {
     if psql -c "" $DB_NAME; then
         echo "$DB_NAME database exists!"
 
-        pg_dump $DB_NAME > ${BACKUPS_FOLDER}/${DB_NAME}-backup-$(date "+%Y%m%d%H%M%S").sql
-        echo "$DB_NAME database backup created."
+        mkdir -p $BACKUPS_FOLDER
+        local BACKUP_FILE=$BACKUPS_FOLDER/$DB_NAME-backup-$(date "+%Y%m%d%H%M%S").sql
+
+        pg_dump $DB_NAME > $BACKUP_FILE
+        chown -f aether:aether $BACKUP_FILE
+        echo "$DB_NAME database backup created in [$BACKUP_FILE]."
     fi
 }
 
@@ -137,6 +141,7 @@ function setup {
     cp /var/tmp/REVISION ${STATIC_DIR}/REVISION 2>/dev/null || true
 
     ./manage.py collectstatic --noinput --verbosity 0
+    chown -Rf aether:aether ${STATIC_ROOT}
     chmod -R 755 ${STATIC_ROOT}
 }
 
@@ -178,6 +183,8 @@ case "$1" in
 
     manage )
         ./manage.py "${@:2}"
+        # required to change migration files owner
+        chown -Rf aether:aether *
     ;;
 
     pip_freeze )

aether-producer/aether/producer/__init__.py

@@ -145,20 +145,20 @@ def broker_info(self):
             for t in iter(md.topics.values()):
                 topics = []
 
-                msg_t = (
-                    f'{t} with {len(t.partitions)} partition(s)'
-                    (f', error: {t.error}' if t.error is not None else '')
-                )
+                msg_t = ', '.join([
+                    f'{str(t)} with {len(t.partitions)} partition(s)',
+                    (f'error: {t.error}' if t.error is not None else ''),
+                ])
                 topics.append(msg_t)
 
                 for p in iter(t.partitions.values()):
-                    msg_p = (
-                        f'partition {p.id}'
-                        f', leader: {p.leader}'
-                        f', replicas: {p.replicas}'
-                        f', isrs: {p.isrs}'
-                        (f', error: {p.error}' if p.error is not None else '')
-                    )
+                    msg_p = ', '.join([
+                        f'partition: {p.id}',
+                        f'leader: {p.leader}',
+                        f'replicas: {p.replicas}',
+                        f'isrs: {p.isrs}',
+                        (f'error: {p.error}' if p.error is not None else ''),
+                    ])
                     topics.append(msg_p)
 
                 res['topics'].append(topics)