Merge branch 'develop' into release-2.2

aether-client-library/conf/pip/requirements.txt

@@ -5,6 +5,7 @@
 ################################################################################
 
 bravado
+jsonschema[format]<4
 requests[security]
 requests_oauthlib
 

aether-client-library/setup.py

@@ -40,6 +40,7 @@ def read(f):
 
     install_requires=[
         'bravado',
+        'jsonschema[format]<4',
         'requests[security]',
         'requests_oauthlib'
     ],

aether-entity-extraction-module/conf/pip/requirements.txt

@@ -16,19 +16,19 @@ aether.python==1.3.0
 attrs==21.2.0
 avro-python3==1.10.2
 certifi==2021.5.30
-charset-normalizer==2.0.4
-coverage==5.5
+charset-normalizer==2.0.6
+coverage==6.0
 decorator==5.1.0
 eha-jsonpath==0.6.0
 fakeredis==1.6.1
 flake8==3.9.2
 flake8-quotes==3.3.0
 gevent==21.8.0
-greenlet==1.1.1
+greenlet==1.1.2
 idna==3.2
 iniconfig==1.1.1
 jsonpath-ng==1.5.3
-jsonschema==3.2.0
+jsonschema==4.0.1
 mccabe==0.6.1
 packaging==21.0
 pluggy==1.0.0
@@ -46,6 +46,6 @@ sortedcontainers==2.4.0
 spavro==1.1.24
 tblib==1.7.0
 toml==0.10.2
-urllib3==1.26.6
+urllib3==1.26.7
 zope.event==4.5.0
 zope.interface==5.4.0

aether-kernel/aether/kernel/api/entity_extractor.py

@@ -107,34 +107,38 @@ def extract(self, request, pk, *args, **kwargs):
         )
 
 
-@api_view(['POST'])
+@api_view(['GET', 'POST'])
 @permission_classes([IsAdminUser])
 @renderer_classes([JSONRenderer])
 def extract_view(request, *args, **kwargs):
     '''
-    Send to redis the submissions that are not yet extracted and
-    whose last modification time was not before the indicated delta (1 day).
+    Submit to redis the submissions that are not yet extracted and
+    whose last modification time was before the indicated delta (1 day).
 
-    Reachable at ``POST /admin/~extract?delta=1d``
+    Reachable at ``GET|POST /admin/~extract?delta=1d&[submit]``
     '''
 
     delta = request.query_params.get('delta', '1d')
     modified = parse_delta(delta)
-    submissions = (
-        Submission.objects
-        .filter(modified__lte=modified)
-        .filter(is_extracted=False)
+    submissions = Submission \
+        .objects \
+        .filter(modified__gte=modified) \
+        .filter(project__active=True) \
         .order_by('-modified')
-        .iterator())
-    count = 0
-    for submission in submissions:
-        count = count + 1
-        send_model_item_to_redis(submission)
+    pending = submissions.filter(is_extracted=False)
+
+    submitted = False
+    if request.method == 'POST' or 'submit' in request.query_params:
+        submitted = True
+        for submission in pending.iterator():
+            send_model_item_to_redis(submission)
 
     return Response(data={
-        'count': count,
+        'count': pending.count(),
+        'total': submissions.count(),
         'delta': delta,
         'modified': modified,
+        'submitted': submitted,
         'timestamp': now(),
     })
 

aether-kernel/aether/kernel/api/serializers.py

@@ -515,6 +515,7 @@ class ProjectStatsSerializer(DynamicFieldsMixin, DynamicFieldsModelSerializer):
     first_submission = serializers.DateTimeField(read_only=True)
     last_submission = serializers.DateTimeField(read_only=True)
     submissions_count = serializers.IntegerField(read_only=True)
+    pending_submissions_count = serializers.IntegerField(read_only=True)
     attachments_count = serializers.IntegerField(read_only=True)
     entities_count = serializers.IntegerField(read_only=True)
 
@@ -523,7 +524,8 @@ class Meta:
         fields = (
             'id', 'name', 'created', 'active',
             'first_submission', 'last_submission',
-            'submissions_count', 'attachments_count', 'entities_count',
+            'submissions_count', 'pending_submissions_count',
+            'attachments_count', 'entities_count',
         )
 
 
@@ -532,6 +534,7 @@ class MappingSetStatsSerializer(DynamicFieldsMixin, DynamicFieldsModelSerializer
     first_submission = serializers.DateTimeField(read_only=True)
     last_submission = serializers.DateTimeField(read_only=True)
     submissions_count = serializers.IntegerField(read_only=True)
+    pending_submissions_count = serializers.IntegerField(read_only=True)
     attachments_count = serializers.IntegerField(read_only=True)
     entities_count = serializers.IntegerField(read_only=True)
 
@@ -540,7 +543,8 @@ class Meta:
         fields = (
             'id', 'name', 'created',
             'first_submission', 'last_submission',
-            'submissions_count', 'attachments_count', 'entities_count',
+            'submissions_count', 'pending_submissions_count',
+            'attachments_count', 'entities_count',
         )
 
 

aether-kernel/aether/kernel/api/tests/test_entity_extractor.py

@@ -262,11 +262,31 @@ def test_admin_extract__endpoint(self):
         self.assertEqual(self.mappingset.submissions.count(), 101)
         self.assertEqual(self.mappingset.submissions.filter(is_extracted=False).count(), 101)
 
-        # request extraction
+        # check extraction with GET
+        response = self.client.get(url + '?delta=1w')
+        data = response.json()
+        self.assertEqual(data['delta'], '1w')
+        self.assertFalse(data['submitted'])
+        self.assertEqual(data['count'], 101)
+
+        # request extraction with POST
+        with mock.patch('aether.kernel.api.entity_extractor.send_model_item_to_redis') as mock_fn_1:
+            response = self.client.post(url + '?delta=1w')
+
+        mock_fn_1.assert_called()
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(data['delta'], '1w')
+        self.assertTrue(data['submitted'])
+        self.assertEqual(data['count'], 101)
+
+        # force extraction with GET
         with mock.patch('aether.kernel.api.entity_extractor.send_model_item_to_redis') as mock_fn_1:
-            response = self.client.post(url + '?delta=0microseconds')
+            response = self.client.get(url + '?delta=1w&submit')
 
         mock_fn_1.assert_called()
         self.assertEqual(response.status_code, 200)
         data = response.json()
-        self.assertEqual(data['delta'], '0microseconds')
+        self.assertEqual(data['delta'], '1w')
+        self.assertTrue(data['submitted'])
+        self.assertEqual(data['total'], 101)

aether-kernel/aether/kernel/api/tests/test_views.py

@@ -185,27 +185,31 @@ def test_project_stats_view(self):
         data = response.json()
         self.assertEqual(data['submissions_count'], submissions_count)
 
-        # let's try again but with an unexistent family
+        # let's try again but with an inexistent family
         response = self.client.get(f'{url}?family=unknown')
         data = response.json()
         self.assertEqual(data['submissions_count'], submissions_count)
+        self.assertEqual(data['pending_submissions_count'], submissions_count)
 
         # let's try with using the project id
         response = self.client.get(f'{url}?family={str(self.project.pk)}')
         data = response.json()
         self.assertEqual(data['submissions_count'], submissions_count)
+        self.assertEqual(data['pending_submissions_count'], submissions_count)
 
         # let's try with the passthrough filter
         response = self.client.get(f'{url}?passthrough=true')
         data = response.json()
         self.assertEqual(data['submissions_count'], submissions_count)
+        self.assertEqual(data['pending_submissions_count'], submissions_count)
 
-        # delete the submissions and check the entities
+        # delete the submissions and check the count
         models.Submission.objects.all().delete()
         self.assertEqual(models.Submission.objects.count(), 0)
         response = self.client.get(url)
         data = response.json()
         self.assertEqual(data['submissions_count'], 0)
+        self.assertEqual(data['pending_submissions_count'], 0)
 
     def test_project_stats_view_fields(self):
         url = reverse('projects_stats-detail', kwargs={'pk': self.project.pk})
@@ -219,7 +223,16 @@ def test_project_stats_view_fields(self):
 
         response = self.client.get(
             url,
-            {'omit': 'created,first_submission,last_submission,submissions_count,entities_count'}
+            {
+                'omit': ','.join([
+                    'created',
+                    'first_submission',
+                    'last_submission',
+                    'submissions_count',
+                    'pending_submissions_count',
+                    'entities_count',
+                ])
+            }
         )
         data = response.json()
         self.assertIn('id', data)
@@ -229,6 +242,7 @@ def test_project_stats_view_fields(self):
         self.assertNotIn('first_submission', data)
         self.assertNotIn('last_submission', data)
         self.assertNotIn('submissions_count', data)
+        self.assertNotIn('pending_submissions_count', data)
         self.assertNotIn('entities_count', data)
 
         response = self.client.get(url, {'fields': 'entities_count'})
@@ -240,6 +254,7 @@ def test_project_stats_view_fields(self):
         self.assertNotIn('first_submission', data)
         self.assertNotIn('last_submission', data)
         self.assertNotIn('submissions_count', data)
+        self.assertNotIn('pending_submissions_count', data)
         self.assertIn('entities_count', data)
 
         response = self.client.get(
@@ -256,6 +271,7 @@ def test_project_stats_view_fields(self):
         self.assertNotIn('first_submission', data)
         self.assertNotIn('last_submission', data)
         self.assertNotIn('submissions_count', data)
+        self.assertNotIn('pending_submissions_count', data)
         self.assertIn('entities_count', data)
 
     def test_mapping_set_stats_view(self):
@@ -266,6 +282,7 @@ def test_mapping_set_stats_view(self):
         self.assertEqual(data['id'], str(self.mappingset.pk))
         submissions_count = models.Submission.objects.filter(mappingset=self.mappingset.pk).count()
         self.assertEqual(data['submissions_count'], submissions_count)
+        self.assertEqual(data['pending_submissions_count'], 0, 'All extracted')
         entities_count = models.Entity.objects.filter(submission__mappingset=self.mappingset.pk).count()
         self.assertEqual(data['entities_count'], entities_count)
         self.assertLessEqual(
@@ -279,6 +296,7 @@ def test_mapping_set_stats_view(self):
         response = self.client.get(url)
         data = response.json()
         self.assertEqual(data['submissions_count'], 0)
+        self.assertEqual(data['pending_submissions_count'], 0)
         self.assertEqual(data['entities_count'], 0)
 
     def test_validate_mappings__success(self):

aether-kernel/aether/kernel/api/views.py

@@ -795,6 +795,14 @@ def _get_group_by_cols(*args):  # pragma: no cover
             qs = qs.annotate(
                 submissions_count=db_models.Count(f'{self.submissions_field}__id', distinct=True)
             )
+        if _is_included('pending_submissions_count'):
+            qs = qs.annotate(
+                pending_submissions_count=db_models.Count(
+                    f'{self.submissions_field}__id',
+                    filter=db_models.Q(**{f'{self.submissions_field}__is_extracted': False}),
+                    distinct=True,
+                )
+            )
         if _is_included('attachments_count'):
             qs = qs.annotate(
                 attachments_count=db_models.Count(f'{self.submissions_field}__attachments__id', distinct=True)

aether-kernel/conf/pip/requirements.txt

@@ -18,24 +18,23 @@ asgiref==3.4.1
 attrs==21.2.0
 autopep8==1.5.7
 avro-python3==1.10.2
-boto3==1.18.40
-botocore==1.21.40
-cachetools==4.2.2
+boto3==1.18.54
+botocore==1.21.54
+cachetools==4.2.4
 certifi==2021.5.30
-cffi==1.14.6
-charset-normalizer==2.0.4
+charset-normalizer==2.0.6
 configparser==5.0.2
 coreapi==2.3.3
 coreschema==0.0.4
-coverage==5.5
+coverage==6.0
 decorator==5.1.0
-Django==3.2.7
+Django==3.2.8
 django-cacheops==6.0
 django-cleanup==5.2.0
-django-cors-headers==3.8.0
+django-cors-headers==3.9.0
 django-debug-toolbar==3.2.2
-django-dynamic-fixture==3.1.1
-django-filter==2.4.0
+django-dynamic-fixture==3.1.2
+django-filter==21.1
 django-minio-storage==0.3.10
 django-model-utils==4.1.1
 django-postgrespool2==2.0.1
@@ -53,56 +52,55 @@ flake8==3.9.2
 flake8-quotes==3.3.0
 funcy==1.16
 google-api-core==2.0.1
-google-auth==2.0.2
+google-auth==2.2.1
 google-cloud-core==2.0.0
-google-cloud-storage==1.42.1
-google-crc32c==1.1.2
-google-resumable-media==2.0.2
+google-cloud-storage==1.42.3
+google-crc32c==1.2.0
+google-resumable-media==2.0.3
 googleapis-common-protos==1.53.0
 gprof2dot==2021.2.21
-greenlet==1.1.1
+greenlet==1.1.2
 idna==3.2
 inflection==0.5.1
 itypes==1.2.0
-Jinja2==3.0.1
+Jinja2==3.0.2
 jmespath==0.10.0
 jsonpath-ng==1.5.3
-jsonschema==3.2.0
+jsonschema==4.0.1
 lxml==4.6.3
 Markdown==3.3.4
 MarkupSafe==2.0.1
 mccabe==0.6.1
 minio==6.0.2
-openpyxl==3.0.8
+openpyxl==3.0.9
 packaging==21.0
 ply==3.11
 prometheus-client==0.11.0
-protobuf==3.17.3
+protobuf==3.18.0
 psycopg2-binary==2.9.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycodestyle==2.7.0
-pycparser==2.20
 pyflakes==2.3.1
 Pygments==2.10.0
 pyparsing==2.4.7
 pyrsistent==0.18.0
 python-dateutil==2.8.2
 python-json-logger==2.0.2
-pytz==2021.1
+pytz==2021.3
 redis==3.5.3
 requests==2.26.0
 rsa==4.7.2
 ruamel.yaml==0.17.16
 ruamel.yaml.clib==0.2.6
 s3transfer==0.5.0
-sentry-sdk==1.3.1
+sentry-sdk==1.4.3
 six==1.16.0
 spavro==1.1.24
-SQLAlchemy==1.4.23
+SQLAlchemy==1.4.25
 sqlparse==0.4.2
 tblib==1.7.0
 toml==0.10.2
 uritemplate==3.0.1
-urllib3==1.26.6
+urllib3==1.26.7
 uWSGI==2.0.19.1