chore: prepare release for last developments #880

README.md

@@ -149,7 +149,7 @@ Also check the aether sdk section about [environment variables](https://github.c
 - `LOGGING_FORMATTER`: `json`. The app messages format.
   Possible values: `verbose` or `json`.
 - `LOGGING_LEVEL`: `info` Logging level for app messages.
-  https://docs.python.org/3.7/library/logging.html#levels
+  https://docs.python.org/3.8/library/logging.html#levels
 
 - `DEBUG` Enables debug mode. Is `false` if unset or set to empty string,
   anything else is considered `true`.

aether-kernel/aether/kernel/api/entity_extractor.py

@@ -18,7 +18,10 @@
 
 from django.db import transaction
 
-from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, extract_create_entities
+from aether.python.entity.extractor import (
+    ENTITY_EXTRACTION_ERRORS as KEY,
+    extract_create_entities,
+)
 from . import models
 
 
@@ -30,7 +33,7 @@ def run_entity_extraction(submission, overwrite=False):
         # replace their payloads with the new ones
         submission.entities.all().delete()
         payload = submission.payload
-        del payload[ENTITY_EXTRACTION_ERRORS]
+        payload.pop(KEY, None)
         submission.payload = payload
         submission.is_extracted = False
         submission.save(update_fields=['payload', 'is_extracted'])
@@ -43,41 +46,43 @@ def run_entity_extraction(submission, overwrite=False):
                          .exclude(definition__entities__isnull=True) \
                          .exclude(definition__entities={})
 
+    payload = dict(submission.payload)
     for mapping in mappings:
         # Get the primary key of the schemadecorator
         entity_sd_ids = mapping.definition.get('entities')
         # Get the schema of the schemadecorator
-        schema_decorator = {
+        schema_decorators = {
             name: models.SchemaDecorator.objects.get(pk=_id)
             for name, _id in entity_sd_ids.items()
         }
         schemas = {
-            name: ps.schema.definition
-            for name, ps in schema_decorator.items()
+            name: sd.schema.definition
+            for name, sd in schema_decorators.items()
         }
-        _, entities = extract_create_entities(
-            submission_payload=submission.payload,
+        payload, entities = extract_create_entities(
+            submission_payload=payload,
             mapping_definition=mapping.definition,
             schemas=schemas,
+            mapping_id=mapping.id,
         )
         for entity in entities:
-            schemadecorator_name = entity.schemadecorator_name
-            schemadecorator = schema_decorator[schemadecorator_name]
-            entity_instance = models.Entity(
+            models.Entity(
+                id=entity.id,
                 payload=entity.payload,
                 status=entity.status,
-                schemadecorator=schemadecorator,
+                schemadecorator=schema_decorators[entity.schemadecorator_name],
                 submission=submission,
                 mapping=mapping,
-                mapping_revision=mapping.revision
-            )
-            entity_instance.save()
+                mapping_revision=mapping.revision,
+                project=submission.project,
+            ).save()
 
     # this should include in the submission payload the following properties
     # generated during the extraction:
     # - ``aether_errors``, with all the errors that made not possible
     #   to create the entities.
     # - ``aether_extractor_enrichment``, with the generated values that allow us
     #   to re-execute this process again with the same result.
+    submission.payload = payload
     submission.is_extracted = submission.entities.count() > 0
     submission.save(update_fields=['payload', 'is_extracted'])

aether-kernel/aether/kernel/api/serializers.py

@@ -35,8 +35,8 @@
 from aether.python import utils
 from aether.python.constants import MergeOptions as MERGE_OPTIONS
 from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS
-from .entity_extractor import run_entity_extraction
 
+from .entity_extractor import run_entity_extraction
 from . import models, validators
 
 
@@ -204,7 +204,7 @@ def create(self, validated_data):
         instance = super(SubmissionSerializer, self).create(validated_data)
         try:
             run_entity_extraction(instance)
-        except Exception as e:
+        except Exception as e:  # pragma: no cover
             instance.payload[ENTITY_EXTRACTION_ERRORS] = instance.payload.get(ENTITY_EXTRACTION_ERRORS, [])
             instance.payload[ENTITY_EXTRACTION_ERRORS] += [str(e)]
             instance.save()

aether-kernel/aether/kernel/api/tests/test_exporter.py

@@ -383,7 +383,7 @@ def setUp(self):
             EXAMPLE_SCHEMA = json.load(infile)
 
         with open(os.path.join(here, 'files/export.json'), 'rb') as infile:
-            EXAMPLE_PAYLOAD = json.load(infile)
+            self.EXAMPLE_PAYLOAD = json.load(infile)
 
         project = models.Project.objects.create(
             name='project1',
@@ -400,7 +400,7 @@ def setUp(self):
             }],
         )
         submission = models.Submission.objects.create(
-            payload=EXAMPLE_PAYLOAD,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=models.MappingSet.objects.get(pk=artefacts_id),
         )
         # extract entities
@@ -714,11 +714,11 @@ def test__generate__xlsx__paginate(self):
 
         submission_1 = models.Submission.objects.first()
         submission_2 = models.Submission.objects.create(
-            payload=submission_1.payload,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=submission_1.mappingset,
         )
         submission_3 = models.Submission.objects.create(
-            payload=submission_1.payload,
+            payload=dict(self.EXAMPLE_PAYLOAD),
             mappingset=submission_1.mappingset,
         )
 
@@ -841,7 +841,7 @@ def test_submissions_export__xlsx__error(self, *args):
     def test_submissions_export__csv__error(self, *args):
         for i in range(13):
             models.Submission.objects.create(
-                payload={'name': f'Person-{i}'},
+                payload=dict({'name': f'Person-{i}'}),
                 mappingset=models.MappingSet.objects.first(),
             )
 
@@ -1225,6 +1225,7 @@ def test_entities_export__attachments(self):
 
         # new submission with 2 attachments
         submission.pk = None
+        submission.payload = dict(self.EXAMPLE_PAYLOAD)
         submission.save()
         self.assertEqual(models.Submission.objects.count(), 2)
 
@@ -1244,6 +1245,7 @@ def test_entities_export__attachments(self):
 
         # new submission without attachments
         submission.pk = None
+        submission.payload = dict(self.EXAMPLE_PAYLOAD)
         submission.save()
         self.assertEqual(models.Submission.objects.count(), 3)
         run_entity_extraction(submission)

aether-kernel/aether/kernel/api/tests/test_filters.py

@@ -23,13 +23,20 @@
 from django.test import TestCase, override_settings
 from django.urls import reverse
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, ENTITY_EXTRACTION_ENRICHMENT
+
 from aether.kernel.api import models
 from aether.kernel.api.filters import EntityFilter, SubmissionFilter
 from aether.kernel.api.tests.utils.generators import (
     generate_project,
     generate_random_string,
 )
 
+ENTITY_EXTRACTION_FIELDS = [
+    ENTITY_EXTRACTION_ERRORS,
+    ENTITY_EXTRACTION_ENRICHMENT,
+]
+
 
 @override_settings(MULTITENANCY=False)
 class TestFilters(TestCase):
@@ -570,12 +577,12 @@ def test_submission_filter__by_payload(self):
                 submission_payload = {
                     k: v
                     for k, v in submission['payload'].items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 original_payload = {
                     k: v
                     for k, v in payload.items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 self.assertEqual(submission_payload, original_payload)
         self.assertEqual(submissions_count, filtered_submissions_count)
@@ -606,12 +613,12 @@ def test_submission_filter__by_payload__post(self):
                 submission_payload = {
                     k: v
                     for k, v in submission['payload'].items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 original_payload = {
                     k: v
                     for k, v in payload.items()
-                    if k not in ('aether_errors', 'aether_extractor_enrichment')
+                    if k not in ENTITY_EXTRACTION_FIELDS
                 }
                 self.assertEqual(submission_payload, original_payload)
         self.assertEqual(submissions_count, filtered_submissions_count)

aether-kernel/aether/kernel/api/tests/test_models.py

@@ -142,7 +142,7 @@ def test_models(self):
 
         submission = models.Submission.objects.create(
             revision='a sample revision',
-            payload=EXAMPLE_SOURCE_DATA,
+            payload=dict(EXAMPLE_SOURCE_DATA),
             mappingset=mappingset,
         )
         self.assertNotEqual(models.Submission.objects.count(), 0)
@@ -187,7 +187,7 @@ def test_models(self):
         with self.assertRaises(IntegrityError) as err4:
             models.Entity.objects.create(
                 revision='a sample revision',
-                payload=EXAMPLE_SOURCE_DATA,  # this is the submission payload without ID
+                payload=dict(EXAMPLE_SOURCE_DATA),  # this is the submission payload without ID
                 status='Publishable',
                 schemadecorator=schemadecorator,
             )

aether-kernel/aether/kernel/api/tests/test_serializers.py

@@ -22,6 +22,8 @@
 from django.test import RequestFactory, TestCase
 from rest_framework.serializers import ValidationError
 
+from aether.python.entity.extractor import ENTITY_EXTRACTION_ENRICHMENT
+
 from aether.kernel.api import models, serializers
 
 from . import EXAMPLE_SCHEMA, EXAMPLE_SOURCE_DATA, EXAMPLE_SOURCE_DATA_ENTITY, EXAMPLE_MAPPING
@@ -174,7 +176,7 @@ def test__serializers__create_and_update(self):
         submission = serializers.SubmissionSerializer(
             data={
                 'project': project.data['id'],
-                'payload': EXAMPLE_SOURCE_DATA,
+                'payload': dict(EXAMPLE_SOURCE_DATA),
             },
             context={'request': self.request},
         )
@@ -197,18 +199,17 @@ def test__serializers__create_and_update(self):
         self.assertTrue(submission.is_valid(), submission.errors)
 
         # save the submission and check that no entities were created
-        # and we have aether errors
         self.assertEqual(models.Entity.objects.count(), 0)
         submission.save()
         self.assertEqual(models.Entity.objects.count(), 0)
-        self.assertIn('aether_errors', submission.data['payload'])
+        self.assertFalse(submission.data['is_extracted'])
 
         # check the submission without entity extraction errors
         submission = serializers.SubmissionSerializer(
             data={
                 'mappingset': mappingset.data['id'],
                 'project': project.data['id'],
-                'payload': EXAMPLE_SOURCE_DATA,
+                'payload': dict(EXAMPLE_SOURCE_DATA),
             },
             context={'request': self.request},
         )
@@ -218,7 +219,7 @@ def test__serializers__create_and_update(self):
         self.assertEqual(models.Entity.objects.count(), 0)
         submission.save()
         self.assertNotEqual(models.Entity.objects.count(), 0)
-        self.assertIn('aether_extractor_enrichment', submission.data['payload'])
+        self.assertIn(ENTITY_EXTRACTION_ENRICHMENT, submission.data['payload'])
 
         # check entity
         entity = serializers.EntitySerializer(
@@ -227,7 +228,7 @@ def test__serializers__create_and_update(self):
                 'submission': submission.data['id'],
                 'schemadecorator': schemadecorator.data['id'],
                 'status': 'Pending Approval',
-                'payload': EXAMPLE_SOURCE_DATA,  # has no id
+                'payload': dict(EXAMPLE_SOURCE_DATA),  # has no id
             },
             context={'request': self.request},
         )
@@ -306,7 +307,7 @@ def test__serializers__create_and_update(self):
 
         create_count = 6
         # make objects
-        payloads = [EXAMPLE_SOURCE_DATA_ENTITY for i in range(create_count)]
+        payloads = [dict(EXAMPLE_SOURCE_DATA_ENTITY) for i in range(create_count)]
         for pl in payloads:
             pl.update({'id': str(uuid.uuid4())})
         data = [