diff --git a/README.md b/README.md index 487fe0827..24a792c45 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Also check the aether sdk section about [environment variables](https://github.c - `LOGGING_FORMATTER`: `json`. The app messages format. Possible values: `verbose` or `json`. - `LOGGING_LEVEL`: `info` Logging level for app messages. - https://docs.python.org/3.7/library/logging.html#levels + https://docs.python.org/3.8/library/logging.html#levels - `DEBUG` Enables debug mode. Is `false` if unset or set to empty string, anything else is considered `true`. diff --git a/aether-kernel/aether/kernel/api/entity_extractor.py b/aether-kernel/aether/kernel/api/entity_extractor.py index 27b1329db..d0aeb861f 100644 --- a/aether-kernel/aether/kernel/api/entity_extractor.py +++ b/aether-kernel/aether/kernel/api/entity_extractor.py @@ -18,7 +18,10 @@ from django.db import transaction -from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, extract_create_entities +from aether.python.entity.extractor import ( + ENTITY_EXTRACTION_ERRORS as KEY, + extract_create_entities, +) from . import models @@ -30,7 +33,7 @@ def run_entity_extraction(submission, overwrite=False): # replace their payloads with the new ones submission.entities.all().delete() payload = submission.payload - del payload[ENTITY_EXTRACTION_ERRORS] + payload.pop(KEY, None) submission.payload = payload submission.is_extracted = False submission.save(update_fields=['payload', 'is_extracted']) @@ -43,35 +46,36 @@ def run_entity_extraction(submission, overwrite=False): .exclude(definition__entities__isnull=True) \ .exclude(definition__entities={}) + payload = dict(submission.payload) for mapping in mappings: # Get the primary key of the schemadecorator entity_sd_ids = mapping.definition.get('entities') # Get the schema of the schemadecorator - schema_decorator = { + schema_decorators = { name: models.SchemaDecorator.objects.get(pk=_id) for name, _id in entity_sd_ids.items() } schemas = { - name: ps.schema.definition - for name, ps in schema_decorator.items() + name: sd.schema.definition + for name, sd in schema_decorators.items() } - _, entities = extract_create_entities( - submission_payload=submission.payload, + payload, entities = extract_create_entities( + submission_payload=payload, mapping_definition=mapping.definition, schemas=schemas, + mapping_id=mapping.id, ) for entity in entities: - schemadecorator_name = entity.schemadecorator_name - schemadecorator = schema_decorator[schemadecorator_name] - entity_instance = models.Entity( + models.Entity( + id=entity.id, payload=entity.payload, status=entity.status, - schemadecorator=schemadecorator, + schemadecorator=schema_decorators[entity.schemadecorator_name], submission=submission, mapping=mapping, - mapping_revision=mapping.revision - ) - entity_instance.save() + mapping_revision=mapping.revision, + project=submission.project, + ).save() # this should include in the submission payload the following properties # generated during the extraction: @@ -79,5 +83,6 @@ def run_entity_extraction(submission, overwrite=False): # to create the entities. # - ``aether_extractor_enrichment``, with the generated values that allow us # to re-execute this process again with the same result. + submission.payload = payload submission.is_extracted = submission.entities.count() > 0 submission.save(update_fields=['payload', 'is_extracted']) diff --git a/aether-kernel/aether/kernel/api/serializers.py b/aether-kernel/aether/kernel/api/serializers.py index d59772f07..36b8cc84c 100644 --- a/aether-kernel/aether/kernel/api/serializers.py +++ b/aether-kernel/aether/kernel/api/serializers.py @@ -35,8 +35,8 @@ from aether.python import utils from aether.python.constants import MergeOptions as MERGE_OPTIONS from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS -from .entity_extractor import run_entity_extraction +from .entity_extractor import run_entity_extraction from . import models, validators @@ -204,7 +204,7 @@ def create(self, validated_data): instance = super(SubmissionSerializer, self).create(validated_data) try: run_entity_extraction(instance) - except Exception as e: + except Exception as e: # pragma: no cover instance.payload[ENTITY_EXTRACTION_ERRORS] = instance.payload.get(ENTITY_EXTRACTION_ERRORS, []) instance.payload[ENTITY_EXTRACTION_ERRORS] += [str(e)] instance.save() diff --git a/aether-kernel/aether/kernel/api/tests/test_exporter.py b/aether-kernel/aether/kernel/api/tests/test_exporter.py index ba6faea73..891095443 100644 --- a/aether-kernel/aether/kernel/api/tests/test_exporter.py +++ b/aether-kernel/aether/kernel/api/tests/test_exporter.py @@ -383,7 +383,7 @@ def setUp(self): EXAMPLE_SCHEMA = json.load(infile) with open(os.path.join(here, 'files/export.json'), 'rb') as infile: - EXAMPLE_PAYLOAD = json.load(infile) + self.EXAMPLE_PAYLOAD = json.load(infile) project = models.Project.objects.create( name='project1', @@ -400,7 +400,7 @@ def setUp(self): }], ) submission = models.Submission.objects.create( - payload=EXAMPLE_PAYLOAD, + payload=dict(self.EXAMPLE_PAYLOAD), mappingset=models.MappingSet.objects.get(pk=artefacts_id), ) # extract entities @@ -714,11 +714,11 @@ def test__generate__xlsx__paginate(self): submission_1 = models.Submission.objects.first() submission_2 = models.Submission.objects.create( - payload=submission_1.payload, + payload=dict(self.EXAMPLE_PAYLOAD), mappingset=submission_1.mappingset, ) submission_3 = models.Submission.objects.create( - payload=submission_1.payload, + payload=dict(self.EXAMPLE_PAYLOAD), mappingset=submission_1.mappingset, ) @@ -841,7 +841,7 @@ def test_submissions_export__xlsx__error(self, *args): def test_submissions_export__csv__error(self, *args): for i in range(13): models.Submission.objects.create( - payload={'name': f'Person-{i}'}, + payload=dict({'name': f'Person-{i}'}), mappingset=models.MappingSet.objects.first(), ) @@ -1225,6 +1225,7 @@ def test_entities_export__attachments(self): # new submission with 2 attachments submission.pk = None + submission.payload = dict(self.EXAMPLE_PAYLOAD) submission.save() self.assertEqual(models.Submission.objects.count(), 2) @@ -1244,6 +1245,7 @@ def test_entities_export__attachments(self): # new submission without attachments submission.pk = None + submission.payload = dict(self.EXAMPLE_PAYLOAD) submission.save() self.assertEqual(models.Submission.objects.count(), 3) run_entity_extraction(submission) diff --git a/aether-kernel/aether/kernel/api/tests/test_filters.py b/aether-kernel/aether/kernel/api/tests/test_filters.py index 8b46d335a..0080f32e1 100644 --- a/aether-kernel/aether/kernel/api/tests/test_filters.py +++ b/aether-kernel/aether/kernel/api/tests/test_filters.py @@ -23,6 +23,8 @@ from django.test import TestCase, override_settings from django.urls import reverse +from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS, ENTITY_EXTRACTION_ENRICHMENT + from aether.kernel.api import models from aether.kernel.api.filters import EntityFilter, SubmissionFilter from aether.kernel.api.tests.utils.generators import ( @@ -30,6 +32,11 @@ generate_random_string, ) +ENTITY_EXTRACTION_FIELDS = [ + ENTITY_EXTRACTION_ERRORS, + ENTITY_EXTRACTION_ENRICHMENT, +] + @override_settings(MULTITENANCY=False) class TestFilters(TestCase): @@ -570,12 +577,12 @@ def test_submission_filter__by_payload(self): submission_payload = { k: v for k, v in submission['payload'].items() - if k not in ('aether_errors', 'aether_extractor_enrichment') + if k not in ENTITY_EXTRACTION_FIELDS } original_payload = { k: v for k, v in payload.items() - if k not in ('aether_errors', 'aether_extractor_enrichment') + if k not in ENTITY_EXTRACTION_FIELDS } self.assertEqual(submission_payload, original_payload) self.assertEqual(submissions_count, filtered_submissions_count) @@ -606,12 +613,12 @@ def test_submission_filter__by_payload__post(self): submission_payload = { k: v for k, v in submission['payload'].items() - if k not in ('aether_errors', 'aether_extractor_enrichment') + if k not in ENTITY_EXTRACTION_FIELDS } original_payload = { k: v for k, v in payload.items() - if k not in ('aether_errors', 'aether_extractor_enrichment') + if k not in ENTITY_EXTRACTION_FIELDS } self.assertEqual(submission_payload, original_payload) self.assertEqual(submissions_count, filtered_submissions_count) diff --git a/aether-kernel/aether/kernel/api/tests/test_models.py b/aether-kernel/aether/kernel/api/tests/test_models.py index a08f73a65..88b85913a 100644 --- a/aether-kernel/aether/kernel/api/tests/test_models.py +++ b/aether-kernel/aether/kernel/api/tests/test_models.py @@ -142,7 +142,7 @@ def test_models(self): submission = models.Submission.objects.create( revision='a sample revision', - payload=EXAMPLE_SOURCE_DATA, + payload=dict(EXAMPLE_SOURCE_DATA), mappingset=mappingset, ) self.assertNotEqual(models.Submission.objects.count(), 0) @@ -187,7 +187,7 @@ def test_models(self): with self.assertRaises(IntegrityError) as err4: models.Entity.objects.create( revision='a sample revision', - payload=EXAMPLE_SOURCE_DATA, # this is the submission payload without ID + payload=dict(EXAMPLE_SOURCE_DATA), # this is the submission payload without ID status='Publishable', schemadecorator=schemadecorator, ) diff --git a/aether-kernel/aether/kernel/api/tests/test_serializers.py b/aether-kernel/aether/kernel/api/tests/test_serializers.py index 845059654..51e89a536 100644 --- a/aether-kernel/aether/kernel/api/tests/test_serializers.py +++ b/aether-kernel/aether/kernel/api/tests/test_serializers.py @@ -22,6 +22,8 @@ from django.test import RequestFactory, TestCase from rest_framework.serializers import ValidationError +from aether.python.entity.extractor import ENTITY_EXTRACTION_ENRICHMENT + from aether.kernel.api import models, serializers from . import EXAMPLE_SCHEMA, EXAMPLE_SOURCE_DATA, EXAMPLE_SOURCE_DATA_ENTITY, EXAMPLE_MAPPING @@ -174,7 +176,7 @@ def test__serializers__create_and_update(self): submission = serializers.SubmissionSerializer( data={ 'project': project.data['id'], - 'payload': EXAMPLE_SOURCE_DATA, + 'payload': dict(EXAMPLE_SOURCE_DATA), }, context={'request': self.request}, ) @@ -197,18 +199,17 @@ def test__serializers__create_and_update(self): self.assertTrue(submission.is_valid(), submission.errors) # save the submission and check that no entities were created - # and we have aether errors self.assertEqual(models.Entity.objects.count(), 0) submission.save() self.assertEqual(models.Entity.objects.count(), 0) - self.assertIn('aether_errors', submission.data['payload']) + self.assertFalse(submission.data['is_extracted']) # check the submission without entity extraction errors submission = serializers.SubmissionSerializer( data={ 'mappingset': mappingset.data['id'], 'project': project.data['id'], - 'payload': EXAMPLE_SOURCE_DATA, + 'payload': dict(EXAMPLE_SOURCE_DATA), }, context={'request': self.request}, ) @@ -218,7 +219,7 @@ def test__serializers__create_and_update(self): self.assertEqual(models.Entity.objects.count(), 0) submission.save() self.assertNotEqual(models.Entity.objects.count(), 0) - self.assertIn('aether_extractor_enrichment', submission.data['payload']) + self.assertIn(ENTITY_EXTRACTION_ENRICHMENT, submission.data['payload']) # check entity entity = serializers.EntitySerializer( @@ -227,7 +228,7 @@ def test__serializers__create_and_update(self): 'submission': submission.data['id'], 'schemadecorator': schemadecorator.data['id'], 'status': 'Pending Approval', - 'payload': EXAMPLE_SOURCE_DATA, # has no id + 'payload': dict(EXAMPLE_SOURCE_DATA), # has no id }, context={'request': self.request}, ) @@ -306,7 +307,7 @@ def test__serializers__create_and_update(self): create_count = 6 # make objects - payloads = [EXAMPLE_SOURCE_DATA_ENTITY for i in range(create_count)] + payloads = [dict(EXAMPLE_SOURCE_DATA_ENTITY) for i in range(create_count)] for pl in payloads: pl.update({'id': str(uuid.uuid4())}) data = [ diff --git a/aether-kernel/aether/kernel/api/tests/test_utils.py b/aether-kernel/aether/kernel/api/tests/test_utils.py index 00d2edb42..2319caf58 100644 --- a/aether-kernel/aether/kernel/api/tests/test_utils.py +++ b/aether-kernel/aether/kernel/api/tests/test_utils.py @@ -41,7 +41,7 @@ def setUp(self): username = 'test' email = 'test@example.com' password = 'testtest' - self.user = get_user_model().objects.create_user(username, email, password) + get_user_model().objects.create_user(username, email, password) self.assertTrue(self.client.login(username=username, password=password)) self.project = models.Project.objects.create( @@ -49,16 +49,16 @@ def setUp(self): name='a project name', ) - url = reverse('project-artefacts', kwargs={'pk': self.project.pk}) + self.mappingset_id = str(uuid.uuid4()) + self.mapping_1 = str(uuid.uuid4()) + self.mapping_2 = str(uuid.uuid4()) - mappingset_id = uuid.uuid4() - - data = { + artefacts = { 'mappingsets': [{ + 'id': self.mappingset_id, 'name': 'Test Mappingset', 'input': EXAMPLE_SOURCE_DATA_WITH_LOCATION, 'schema': {}, - 'id': mappingset_id, }], 'schemas': [ { @@ -72,89 +72,63 @@ def setUp(self): { 'name': 'Person', 'definition': EXAMPLE_SCHEMA, - } + }, ], 'mappings': [ { + 'id': self.mapping_1, 'name': 'mapping-1', 'definition': { 'mapping': EXAMPLE_FIELD_MAPPINGS, }, 'is_active': True, 'is_ready_only': False, - 'mappingset': mappingset_id, + 'mappingset': self.mappingset_id, }, { + 'id': self.mapping_2, 'name': 'mapping-2', 'definition': { 'mapping': EXAMPLE_FIELD_MAPPINGS_LOCATION, }, 'is_active': True, 'is_ready_only': False, - 'mappingset': mappingset_id, - } + 'mappingset': self.mappingset_id, + }, ], } - self.project_artefacts = self.client.patch( - url, - data=data, + self.client.patch( + reverse('project-artefacts', kwargs={'pk': self.project.pk}), + data=artefacts, content_type='application/json', - ).json() + ) def tearDown(self): self.project.delete() self.client.logout() def test_get_unique_schemas_used(self): - url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]}) - mapping = self.client.get(url).json() - if mapping['name'] == 'mapping-1': - mapping_1 = mapping - self.mapping = mapping['id'] - else: - mapping_2 = mapping - - url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][1]}) - mapping = self.client.get(url).json() - if mapping['name'] == 'mapping-2': - mapping_2 = mapping - else: - mapping_1 = mapping - self.mapping = mapping['id'] - - self.assertEqual(mapping_2['name'], 'mapping-2') - self.assertEqual(mapping_1['name'], 'mapping-1') - - result = utils.get_unique_schemas_used([mapping_1['id']]) + result = utils.get_unique_schemas_used([self.mapping_1]) self.assertEqual(len(result), 1) self.assertEqual(next(iter(result)), 'Person') self.assertFalse(result[next(iter(result))]['is_unique']) - result = utils.get_unique_schemas_used([mapping_2['id']]) + result = utils.get_unique_schemas_used([self.mapping_2]) self.assertEqual(len(result), 2) self.assertFalse(result['Person']['is_unique']) self.assertTrue(result['Location']['is_unique']) - result = utils.get_unique_schemas_used([mapping_2['id'], mapping_1['id']]) + result = utils.get_unique_schemas_used([self.mapping_1, self.mapping_2]) self.assertEqual(len(result), 2) self.assertTrue(result['Person']['is_unique']) self.assertTrue(result['Location']['is_unique']) def test_bulk_delete_by_mappings_mapping(self): - url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]}) - mapping = self.client.get(url).json() - if mapping['name'] == 'mapping-2': - mapping_2 = mapping - else: - url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][1]}) - mapping_2 = self.client.get(url).json() - - self.assertEqual(mapping_2['name'], 'mapping-2') opts = { 'entities': True, 'schemas': True, } - result = utils.bulk_delete_by_mappings(opts, None, [mapping_2['id']]) + result = utils.bulk_delete_by_mappings(opts, None, [self.mapping_2]) self.assertFalse(result['schemas']['Person']['is_unique']) self.assertTrue(result['schemas']['Location']['is_unique']) self.assertTrue(result['schemas']['Location']['is_deleted']) @@ -162,16 +136,12 @@ def test_bulk_delete_by_mappings_mapping(self): self.assertNotIn('submissions', result) def test_bulk_delete_by_mappings_mappingset(self): - url = reverse('mapping-detail', kwargs={'pk': self.project_artefacts['mappings'][0]}) - mapping = self.client.get(url).json() - mapping_object = models.Mapping.objects.get(pk=mapping['id']) - mappingset = mapping_object.mappingset.id opts = { 'entities': True, 'schemas': True, 'submissions': True } - result = utils.bulk_delete_by_mappings(opts, mappingset) + result = utils.bulk_delete_by_mappings(opts, self.mappingset_id) self.assertTrue(result['schemas']['Person']['is_unique']) self.assertTrue(result['schemas']['Location']['is_unique']) self.assertTrue(result['schemas']['Location']['is_deleted']) @@ -184,32 +154,30 @@ def test_bulk_delete_by_mappings_mappingset(self): 'schemas': False, 'submissions': False } - result = utils.bulk_delete_by_mappings(opts, mappingset) + result = utils.bulk_delete_by_mappings(opts, self.mappingset_id) self.assertEqual(result, {}) def test_bulk_delete_by_mappings_with_submissions(self): - mapping_object = models.Mapping.objects.get(pk=self.project_artefacts['mappings'][0]) - mappingset = mapping_object.mappingset.id - - url = reverse('submission-list') - data = { - 'payload': EXAMPLE_SOURCE_DATA_WITH_LOCATION, + submission = { + 'payload': dict(EXAMPLE_SOURCE_DATA_WITH_LOCATION), 'project': str(self.project.id), - 'mappingset': mappingset + 'mappingset': self.mappingset_id, } self.client.post( - url, - data=data, + reverse('submission-list'), + data=submission, content_type='application/json', ) + entity_count = models.Entity.objects.filter( + mapping__id__in=[self.mapping_1, self.mapping_2] + ).count() + self.assertTrue(entity_count > 0) + opts = { 'entities': True, 'submissions': True } - entity_count = models.Entity.objects.filter( - mapping__id__in=self.project_artefacts['mappings'] - ).count() - result = utils.bulk_delete_by_mappings(opts, mappingset) + result = utils.bulk_delete_by_mappings(opts, self.mappingset_id) self.assertEqual(result['entities']['total'], entity_count) self.assertTrue(result['entities']['schemas']) self.assertEqual(result['entities']['schemas'][0]['name'], 'Person') diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py index 180dc1499..7c95ac848 100644 --- a/aether-kernel/aether/kernel/api/tests/test_views.py +++ b/aether-kernel/aether/kernel/api/tests/test_views.py @@ -26,10 +26,10 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase, override_settings from django.urls import reverse -from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS - from rest_framework import status +from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS + from aether.kernel.api import models from aether.kernel.api.entity_extractor import run_entity_extraction from aether.kernel.api.tests.utils.generators import generate_project @@ -99,7 +99,7 @@ def setUp(self): ) self.submission = models.Submission.objects.create( - payload=EXAMPLE_SOURCE_DATA, + payload=dict(EXAMPLE_SOURCE_DATA), mappingset=self.mappingset, project=self.project, ) @@ -159,7 +159,7 @@ def test_project_stats_view(self): # this will also trigger the entities extraction # (4 entities per submission -> 3 for self.schemadecorator + 1 for schemadecorator_2) self.helper_create_object('submission-list', { - 'payload': EXAMPLE_SOURCE_DATA, + 'payload': dict(EXAMPLE_SOURCE_DATA), 'mappingset': str(self.mappingset.pk), }) @@ -772,7 +772,7 @@ def test_submission__extract__endpoint(self): models.Entity.objects.all().delete() # remove all entities self.assertEqual(self.submission.entities.count(), 0) self.submission.refresh_from_db() - self.assertEqual(self.submission.payload['aether_errors'], []) + self.assertEqual(self.submission.payload[ENTITY_EXTRACTION_ERRORS], []) response = self.client.post(url) self.assertEqual(response.status_code, 405, 'only PATCH') @@ -783,11 +783,29 @@ def test_submission__extract__endpoint(self): self.assertEqual(response.status_code, 400) self.assertEqual(self.submission.entities.count(), 0) self.submission.refresh_from_db() - self.assertEqual(self.submission.payload['aether_errors'], ['oops']) + self.assertEqual(self.submission.payload[ENTITY_EXTRACTION_ERRORS], ['oops']) response = self.client.patch(url) self.assertEqual(response.status_code, 200) - self.assertNotEqual(self.submission.entities.count(), 0) + entities_count = self.submission.entities.count() + self.assertNotEqual(entities_count, 0) + entity_ids = [e.id for e in self.submission.entities.all()] + + # re-extract (same number of entities with the same IDs) + models.Entity.objects.all().delete() # remove all entities + self.client.patch(url) + self.assertEqual(entities_count, self.submission.entities.count()) + self.assertEqual(entity_ids, [e.id for e in self.submission.entities.all()]) + + # re-extract (no new Entities just updated) + for e in self.submission.entities.all(): + e.status = 'Pending Approval' + e.save() + self.client.patch(url) + self.assertEqual(entities_count, self.submission.entities.count()) + self.assertEqual(entity_ids, [e.id for e in self.submission.entities.all()]) + for e in self.submission.entities.all(): + self.assertEqual(e.status, 'Publishable') def test_schema_unique_usage(self): url = reverse('schema-unique-usage') @@ -919,7 +937,7 @@ def test_submission_validate(self): url = reverse('submission-validate') data = { 'mappingset': str(test_mappingset.id), - 'payload': PAYLOAD + 'payload': dict(PAYLOAD), } response = self.client.post( url, @@ -947,7 +965,7 @@ def test_submission_validate(self): self.assertEqual('Not accessible by this realm', response_data['detail']) del PAYLOAD['facility_name'] - data['payload'] = PAYLOAD + data['payload'] = dict(PAYLOAD) response = self.client.post( url, data=data, @@ -971,7 +989,7 @@ def test_submission_validate(self): data = { 'mappingset': 'wrong-uuid', - 'payload': PAYLOAD + 'payload': dict(PAYLOAD) } response = self.client.post( url, diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py index c3185c60b..862e32d4c 100644 --- a/aether-kernel/aether/kernel/api/views.py +++ b/aether-kernel/aether/kernel/api/views.py @@ -387,10 +387,10 @@ def validate(self, request, *args, **kwargs): expected response: { - # Bool indicating if the submission is valid or not + # flag indicating if the submission is valid or not 'is_valid': True|False, - # list of entities successfully generated from the submitted payload + # list of entities successfully generated from the submitted payload 'entities': [], # list of encountered errors @@ -409,13 +409,11 @@ def validate(self, request, *args, **kwargs): try: mappingset = get_object_or_404(models.MappingSet.objects.all(), pk=mappingset_id) except Exception as e: - return Response( - str(e), - status=status.HTTP_400_BAD_REQUEST, - ) + return Response(str(e), status=status.HTTP_400_BAD_REQUEST) if not self.check_realm_permission(request, mappingset): raise PermissionDenied(_('Not accessible by this realm')) + mappings = mappingset.mappings.all() result = { 'is_valid': True, @@ -432,20 +430,20 @@ def validate(self, request, *args, **kwargs): submission_payload=payload, mapping_definition=mapping.definition, schemas=schemas, + mapping_id=mapping.id, ) if submission_data.get(ENTITY_EXTRACTION_ERRORS): result['is_valid'] = False result[ENTITY_EXTRACTION_ERRORS] += submission_data[ENTITY_EXTRACTION_ERRORS] else: result['entities'] += entities + except Exception as e: # pragma: no cover result['is_valid'] = False result[ENTITY_EXTRACTION_ERRORS].append(str(e)) - return Response(result, status=status.HTTP_500_INTERNAL_SERVER_ERROR) - if result['is_valid']: - return Response(result) - return Response(result, status=status.HTTP_400_BAD_REQUEST) + _status = status.HTTP_200_OK if result['is_valid'] else status.HTTP_400_BAD_REQUEST + return Response(result, status=_status) @action(detail=True, methods=['patch']) def extract(self, request, pk, *args, **kwargs): @@ -823,9 +821,10 @@ def validate_mappings_view(request, *args, **kwargs): def run_mapping_validation(submission_payload, mapping_definition, schemas): submission_data, entities = extract_create_entities( - submission_payload, - mapping_definition, - schemas, + submission_payload=submission_payload, + mapping_definition=mapping_definition, + schemas=schemas, + mapping_id='validation', ) validation_result = validate_mappings( submission_payload=submission_payload, diff --git a/aether-kernel/aether/kernel/management/commands/extract_entities.py b/aether-kernel/aether/kernel/management/commands/extract_entities.py index 8535b509f..79ebc11aa 100644 --- a/aether-kernel/aether/kernel/management/commands/extract_entities.py +++ b/aether-kernel/aether/kernel/management/commands/extract_entities.py @@ -21,9 +21,9 @@ from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ +from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS from aether.kernel.api.models import Submission, Entity from aether.kernel.api.entity_extractor import run_entity_extraction -from aether.python.entity.extractor import ENTITY_EXTRACTION_ERRORS class Command(BaseCommand): diff --git a/aether-kernel/aether/kernel/tests/test_commands.py b/aether-kernel/aether/kernel/tests/test_commands.py index 892526012..f1899c003 100644 --- a/aether-kernel/aether/kernel/tests/test_commands.py +++ b/aether-kernel/aether/kernel/tests/test_commands.py @@ -23,7 +23,7 @@ from django.test import TestCase from aether.kernel.api.tests.utils.generators import generate_project -from aether.kernel.api.models import Submission, Entity +from aether.kernel.api.models import Entity class ExtractEntitiesCommandTest(TestCase): @@ -56,16 +56,16 @@ def test__extract_entities__success(self): def test__extract_entities__error(self): generate_project() self.assertNotEqual(Entity.objects.count(), 0) - entities = Entity.objects.count() + entities_count = Entity.objects.count() - for submission in Submission.objects.all(): - submission.payload = {} # make extraction fail - submission.save() + with mock.patch('aether.kernel.api.entity_extractor.extract_create_entities', + side_effect=Exception('oops')) as mock_extractor: + try: + call_command('extract_entities', stdout=self.out, stderr=self.out) + self.assertTrue(True) + except Exception: + self.assertTrue(False) - try: - call_command('extract_entities', stdout=self.out, stderr=self.out) - self.assertTrue(True) - except Exception: - self.assertTrue(False) - self.assertEqual(Entity.objects.count(), entities, + self.assertEqual(Entity.objects.count(), entities_count, 'transaction atomic reverts the deletion') + mock_extractor.assert_called() diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt index 0fe11af6f..d020969f4 100644 --- a/aether-kernel/conf/pip/requirements.txt +++ b/aether-kernel/conf/pip/requirements.txt @@ -12,30 +12,30 @@ # ################################################################################ -aether.python==1.0.17 +aether.python==1.2.0 aether.sdk==1.3.1 attrs==19.3.0 -autopep8==1.5.3 -boto3==1.14.9 -botocore==1.17.9 -cachetools==4.1.0 +autopep8==1.5.4 +boto3==1.14.35 +botocore==1.17.35 +cachetools==4.1.1 certifi==2020.6.20 -cffi==1.14.0 +cffi==1.14.1 chardet==3.0.4 configparser==5.0.0 coreapi==2.3.3 coreschema==0.0.4 -coverage==5.1 -cryptography==2.9.2 +coverage==5.2.1 +cryptography==3.0 decorator==4.4.2 -Django==2.2.13 -django-cacheops==5.0 +Django==2.2.15 +django-cacheops==5.0.1 django-cleanup==5.0.0 django-cors-headers==3.4.0 django-debug-toolbar==2.2 django-dynamic-fixture==3.1.0 django-filter==2.3.0 -django-minio-storage==0.3.7 +django-minio-storage==0.3.8 django-model-utils==4.0.0 django-postgrespool2==1.0.1 django-prometheus==2.0.0 @@ -43,7 +43,7 @@ django-redis==4.12.1 django-silk==4.0.1 django-storages==1.9.1 django-uwsgi==0.2.2 -djangorestframework==3.11.0 +djangorestframework==3.11.1 docutils==0.15.2 drf-dynamic-fields==0.3.1 drf-yasg==1.17.1 @@ -52,15 +52,15 @@ et-xmlfile==1.0.1 flake8==3.8.3 flake8-quotes==3.2.0 funcy==1.14 -google-api-core==1.21.0 -google-auth==1.18.0 -google-cloud-core==1.3.0 -google-cloud-storage==1.29.0 -google-resumable-media==0.5.1 +google-api-core==1.22.0 +google-auth==1.20.0 +google-cloud-core==1.4.0 +google-cloud-storage==1.30.0 +google-crc32c==0.1.0 +google-resumable-media==0.7.0 googleapis-common-protos==1.52.0 gprof2dot==2019.11.30 -idna==2.9 -importlib-metadata==1.6.1 +idna==2.10 inflection==0.5.0 itypes==1.2.0 jdcal==1.4.1 @@ -68,15 +68,15 @@ Jinja2==2.11.2 jmespath==0.10.0 jsonpath-ng==1.5.1 jsonschema==3.2.0 -lxml==4.5.1 +lxml==4.5.2 MarkupSafe==1.1.1 mccabe==0.6.1 -minio==5.0.10 -openpyxl==3.0.3 +minio==6.0.0 +openpyxl==3.0.4 packaging==20.4 ply==3.11 prometheus-client==0.8.0 -protobuf==3.12.2 +protobuf==3.12.4 psycopg2-binary==2.8.5 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -96,14 +96,13 @@ rsa==4.6 ruamel.yaml==0.16.10 ruamel.yaml.clib==0.2.0 s3transfer==0.3.3 -sentry-sdk==0.15.1 +sentry-sdk==0.16.3 six==1.15.0 spavro==1.1.23 -SQLAlchemy==1.3.17 +SQLAlchemy==1.3.18 sqlparse==0.3.1 -tblib==1.6.0 +tblib==1.7.0 toml==0.10.1 uritemplate==3.0.1 -urllib3==1.25.9 +urllib3==1.25.10 uWSGI==2.0.19.1 -zipp==3.1.0 diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt index 2fbea0af3..178eabbad 100644 --- a/aether-odk-module/conf/pip/requirements.txt +++ b/aether-odk-module/conf/pip/requirements.txt @@ -13,52 +13,53 @@ ################################################################################ aether.sdk==1.3.1 -autopep8==1.5.3 -boto3==1.14.9 -botocore==1.17.9 -cachetools==4.1.0 +autopep8==1.5.4 +boto3==1.14.35 +botocore==1.17.35 +cachetools==4.1.1 certifi==2020.6.20 -cffi==1.14.0 +cffi==1.14.1 chardet==3.0.4 configparser==5.0.0 -coverage==5.1 -cryptography==2.9.2 -Django==2.2.13 -django-cacheops==5.0 +coverage==5.2.1 +cryptography==3.0 +Django==2.2.15 +django-cacheops==5.0.1 django-cleanup==5.0.0 django-cors-headers==3.4.0 django-debug-toolbar==2.2 -django-minio-storage==0.3.7 +django-minio-storage==0.3.8 django-postgrespool2==1.0.1 django-prometheus==2.0.0 django-redis==4.12.1 django-silk==4.0.1 django-storages==1.9.1 django-uwsgi==0.2.2 -djangorestframework==3.11.0 +djangorestframework==3.11.1 docutils==0.15.2 drf-dynamic-fields==0.3.1 flake8==3.8.3 flake8-quotes==3.2.0 FormEncode==1.3.1 funcy==1.14 -google-api-core==1.21.0 -google-auth==1.18.0 -google-cloud-core==1.3.0 -google-cloud-storage==1.29.0 -google-resumable-media==0.5.1 +google-api-core==1.22.0 +google-auth==1.20.0 +google-cloud-core==1.4.0 +google-cloud-storage==1.30.0 +google-crc32c==0.1.0 +google-resumable-media==0.7.0 googleapis-common-protos==1.52.0 gprof2dot==2019.11.30 -idna==2.9 +idna==2.10 Jinja2==2.11.2 jmespath==0.10.0 linecache2==1.0.0 -lxml==4.5.1 +lxml==4.5.2 MarkupSafe==1.1.1 mccabe==0.6.1 -minio==5.0.10 +minio==6.0.0 prometheus-client==0.8.0 -protobuf==3.12.2 +protobuf==3.12.4 psycopg2-binary==2.8.5 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -75,16 +76,16 @@ redis==3.5.3 requests==2.24.0 rsa==4.6 s3transfer==0.3.3 -sentry-sdk==0.15.1 +sentry-sdk==0.16.3 six==1.15.0 spavro==1.1.23 -SQLAlchemy==1.3.17 +SQLAlchemy==1.3.18 sqlparse==0.3.1 -tblib==1.6.0 +tblib==1.7.0 toml==0.10.1 traceback2==1.4.0 unicodecsv==0.14.1 unittest2==1.1.0 -urllib3==1.25.9 +urllib3==1.25.10 uWSGI==2.0.19.1 xlrd==1.2.0 diff --git a/aether-producer/aether/producer/__init__.py b/aether-producer/aether/producer/__init__.py index 1170e1c90..cdc85d999 100644 --- a/aether-producer/aether/producer/__init__.py +++ b/aether-producer/aether/producer/__init__.py @@ -130,21 +130,25 @@ def broker_info(self): res['brokers'].append(f'{b}') for t in iter(md.topics.values()): - t_str = [] - t_str.append( + topics = [] + + msg_t = ( f'{t} with {len(t.partitions)} partition(s)' (f', error: {t.error}' if t.error is not None else '') ) + topics.append(msg_t) for p in iter(t.partitions.values()): - t_str.append( + msg_p = ( f'partition {p.id}' f', leader: {p.leader}' f', replicas: {p.replicas}' f', isrs: {p.isrs}' (f', error: {p.error}' if p.error is not None else '') ) - res['topics'].append(t_str) + topics.append(msg_p) + + res['topics'].append(topics) return res except Exception as err: return {'error': f'{err}'} diff --git a/aether-producer/aether/producer/topic.py b/aether-producer/aether/producer/topic.py index 737d3d251..cfddf69b0 100644 --- a/aether-producer/aether/producer/topic.py +++ b/aether-producer/aether/producer/topic.py @@ -36,7 +36,6 @@ from aether.producer.db import Offset from aether.producer.settings import SETTINGS, KAFKA_SETTINGS, get_logger - logger = get_logger('topic') diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt index f55b41408..c472a3dcc 100644 --- a/aether-producer/conf/pip/requirements.txt +++ b/aether-producer/conf/pip/requirements.txt @@ -14,17 +14,18 @@ attrs==19.3.0 certifi==2020.6.20 -cffi==1.14.0 +cffi==1.14.1 chardet==3.0.4 click==7.1.2 -confluent-kafka==1.4.2 -cryptography==2.9.2 +confluent-kafka==1.5.0 +cryptography==3.0 flake8==3.8.3 flake8-quotes==3.2.0 Flask==1.1.2 gevent==20.6.2 greenlet==0.4.16 -idna==2.9 +idna==2.10 +iniconfig==1.0.1 itsdangerous==1.1.0 Jinja2==2.11.2 MarkupSafe==1.1.1 @@ -34,19 +35,19 @@ packaging==20.4 pluggy==0.13.1 psycogreen==1.0.2 psycopg2-binary==2.8.5 -py==1.8.2 +py==1.9.0 pycodestyle==2.6.0 pycparser==2.20 pyflakes==2.2.0 pyOpenSSL==19.1.0 pyparsing==2.4.7 -pytest==5.4.3 +pytest==6.0.1 requests==2.24.0 six==1.15.0 spavro==1.1.23 -SQLAlchemy==1.3.17 -urllib3==1.25.9 -wcwidth==0.2.5 +SQLAlchemy==1.3.18 +toml==0.10.1 +urllib3==1.25.10 Werkzeug==1.0.1 zope.event==4.4 zope.interface==5.1.0 diff --git a/aether-ui/aether/ui/assets/css/base/_fonts.scss b/aether-ui/aether/ui/assets/css/base/_fonts.scss index 91ec9a224..123ed36bd 100644 --- a/aether-ui/aether/ui/assets/css/base/_fonts.scss +++ b/aether-ui/aether/ui/assets/css/base/_fonts.scss @@ -23,5 +23,5 @@ @import url('https://fonts.googleapis.com/css?family=Fira+Mono'); /* Font Awesome 5 https://fontawesome.com/ */ -@import url('https://use.fontawesome.com/releases/v5.13.0/css/fontawesome.css'); -@import url('https://use.fontawesome.com/releases/v5.13.0/css/solid.css'); +@import url('https://use.fontawesome.com/releases/v5.14.0/css/fontawesome.css'); +@import url('https://use.fontawesome.com/releases/v5.14.0/css/solid.css'); diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json index e4faba27d..405b361fc 100644 --- a/aether-ui/aether/ui/assets/package.json +++ b/aether-ui/aether/ui/assets/package.json @@ -29,26 +29,26 @@ "react": "~16.13.0", "react-clipboard.js": "~2.0.16", "react-dom": "~16.13.0", - "react-intl": "~4.7.0", + "react-intl": "~5.4.0", "react-outside-click-handler": "~1.3.0", "react-redux": "~7.2.0", "react-router-dom": "~5.2.0", "redux": "~4.0.0", "redux-thunk": "~2.3.0", - "uuid": "~8.2.0", + "uuid": "~8.3.0", "webpack-google-cloud-storage-plugin": "~0.9.0", "webpack-s3-plugin": "~1.0.3", - "whatwg-fetch": "~3.0.0" + "whatwg-fetch": "~3.3.0" }, "devDependencies": { - "@babel/core": "~7.10.0", + "@babel/core": "~7.11.0", "@babel/plugin-proposal-class-properties": "~7.10.0", - "@babel/preset-env": "~7.10.0", + "@babel/preset-env": "~7.11.0", "@babel/preset-react": "~7.10.0", "@hot-loader/react-dom": "~16.13.0", "babel-loader": "~8.1.0", - "css-loader": "~3.6.0", - "eslint": "~7.3.0", + "css-loader": "~4.2.0", + "eslint": "~7.6.0", "enzyme": "~3.11.0", "enzyme-adapter-react-16": "~1.15.0", "express": "~4.17.0", @@ -59,12 +59,12 @@ "node-sass": "~4.14.0", "react-hot-loader": "~4.12.0", "redux-devtools-extension": "~2.13.0", - "sass-loader": "~8.0.0", + "sass-loader": "~9.0.0", "standard": "~14.3.0", "style-loader": "~1.2.0", "stylelint": "~13.6.0", "stylelint-config-standard": "~20.0.0", - "webpack": "~4.43.0", + "webpack": "~4.44.0", "webpack-bundle-tracker": "~0.4.3", "webpack-cli": "~3.3.0", "webpack-dev-middleware": "~3.7.0", diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt index a7e831cf3..06d672867 100644 --- a/aether-ui/conf/pip/requirements.txt +++ b/aether-ui/conf/pip/requirements.txt @@ -13,22 +13,22 @@ ################################################################################ aether.sdk==1.3.1 -autopep8==1.5.3 -boto3==1.14.9 -botocore==1.17.9 -cachetools==4.1.0 +autopep8==1.5.4 +boto3==1.14.35 +botocore==1.17.35 +cachetools==4.1.1 certifi==2020.6.20 -cffi==1.14.0 +cffi==1.14.1 chardet==3.0.4 configparser==5.0.0 -coverage==5.1 -cryptography==2.9.2 -Django==2.2.13 -django-cacheops==5.0 +coverage==5.2.1 +cryptography==3.0 +Django==2.2.15 +django-cacheops==5.0.1 django-cleanup==5.0.0 django-cors-headers==3.4.0 django-debug-toolbar==2.2 -django-minio-storage==0.3.7 +django-minio-storage==0.3.8 django-model-utils==4.0.0 django-postgrespool2==1.0.1 django-prometheus==2.0.0 @@ -37,27 +37,28 @@ django-silk==4.0.1 django-storages==1.9.1 django-uwsgi==0.2.2 django-webpack-loader==0.7.0 -djangorestframework==3.11.0 +djangorestframework==3.11.1 docutils==0.15.2 drf-dynamic-fields==0.3.1 flake8==3.8.3 flake8-quotes==3.2.0 funcy==1.14 -google-api-core==1.21.0 -google-auth==1.18.0 -google-cloud-core==1.3.0 -google-cloud-storage==1.29.0 -google-resumable-media==0.5.1 +google-api-core==1.22.0 +google-auth==1.20.0 +google-cloud-core==1.4.0 +google-cloud-storage==1.30.0 +google-crc32c==0.1.0 +google-resumable-media==0.7.0 googleapis-common-protos==1.52.0 gprof2dot==2019.11.30 -idna==2.9 +idna==2.10 Jinja2==2.11.2 jmespath==0.10.0 MarkupSafe==1.1.1 mccabe==0.6.1 -minio==5.0.10 +minio==6.0.0 prometheus-client==0.8.0 -protobuf==3.12.2 +protobuf==3.12.4 psycopg2-binary==2.8.5 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -73,11 +74,11 @@ redis==3.5.3 requests==2.24.0 rsa==4.6 s3transfer==0.3.3 -sentry-sdk==0.15.1 +sentry-sdk==0.16.3 six==1.15.0 -SQLAlchemy==1.3.17 +SQLAlchemy==1.3.18 sqlparse==0.3.1 -tblib==1.6.0 +tblib==1.7.0 toml==0.10.1 -urllib3==1.25.9 +urllib3==1.25.10 uWSGI==2.0.19.1 diff --git a/docker-compose-base.yml b/docker-compose-base.yml index 13ad3a756..5903437fe 100644 --- a/docker-compose-base.yml +++ b/docker-compose-base.yml @@ -157,9 +157,9 @@ services: - ./aether-kernel:/code # ------------------------------------------------------------- # to speed up SDK development changes - # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk + # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk # to speed up Aether Python Library development changes - # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/usr/local/lib/python3.7/site-packages/aether/python + # - ${AETHER_PYTHON_PATH:-../aether-python-library}/aether/python:/usr/local/lib/python3.8/site-packages/aether/python command: start_dev @@ -232,7 +232,7 @@ services: - ./aether-odk-module:/code # ------------------------------------------------------------- # to speed up SDK development changes - # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk + # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk command: start_dev @@ -297,7 +297,7 @@ services: - ./aether-ui:/code # ------------------------------------------------------------- # to speed up SDK development changes - # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.7/site-packages/aether/sdk + # - ${SDK_PATH:-../aether-django-sdk-library}/aether/sdk:/usr/local/lib/python3.8/site-packages/aether/sdk command: start_dev ui-assets-base: