From a65936915e10460269698c82e13a5a687c457393 Mon Sep 17 00:00:00 2001 From: obdulia-losantos Date: Mon, 16 Aug 2021 15:20:39 +0200 Subject: [PATCH 1/3] chore: upgrade dependencies --- .../conf/pip/requirements.txt | 26 ++++---- aether-kernel/conf/pip/requirements.txt | 60 +++++++++---------- aether-odk-module/conf/pip/requirements.txt | 52 ++++++++-------- aether-producer/conf/pip/requirements.txt | 24 ++++---- .../aether/ui/assets/css/base/_fonts.scss | 4 +- aether-ui/aether/ui/assets/package.json | 24 ++++---- aether-ui/conf/pip/requirements.txt | 50 ++++++++-------- 7 files changed, 113 insertions(+), 127 deletions(-) diff --git a/aether-entity-extraction-module/conf/pip/requirements.txt b/aether-entity-extraction-module/conf/pip/requirements.txt index ae5c414ec..4db4d1553 100644 --- a/aether-entity-extraction-module/conf/pip/requirements.txt +++ b/aether-entity-extraction-module/conf/pip/requirements.txt @@ -16,40 +16,36 @@ aether.python==1.3.0 attrs==21.2.0 avro-python3==1.10.2 certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +charset-normalizer==2.0.4 coverage==5.5 -cryptography==3.4.7 decorator==5.0.9 eha-jsonpath==0.6.0 -fakeredis==1.5.1 +fakeredis==1.6.0 flake8==3.9.2 flake8-quotes==3.2.0 -gevent==21.1.2 -greenlet==1.1.0 -idna==2.10 +gevent==21.8.0 +greenlet==1.1.1 +idna==3.2 iniconfig==1.1.1 -jsonpath-ng==1.5.2 +jsonpath-ng==1.5.3 jsonschema==3.2.0 mccabe==0.6.1 -packaging==20.9 +packaging==21.0 pluggy==0.13.1 ply==3.11 py==1.10.0 pycodestyle==2.7.0 -pycparser==2.20 pyflakes==2.3.1 -pyOpenSSL==20.0.1 pyparsing==2.4.7 -pyrsistent==0.17.3 +pyrsistent==0.18.0 pytest==6.2.4 redis==3.5.3 -requests==2.25.1 +requests==2.26.0 six==1.16.0 sortedcontainers==2.4.0 -spavro==1.1.23 +spavro==1.1.24 tblib==1.7.0 toml==0.10.2 -urllib3==1.26.5 +urllib3==1.26.6 zope.event==4.5.0 zope.interface==5.4.0 diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt index 17eb665dd..ddd13c0c2 100644 --- a/aether-kernel/conf/pip/requirements.txt +++ b/aether-kernel/conf/pip/requirements.txt @@ -17,23 +17,22 @@ aether.sdk==1.3.8 attrs==21.2.0 autopep8==1.5.7 avro-python3==1.10.2 -boto3==1.17.86 -botocore==1.20.86 +boto3==1.18.21 +botocore==1.21.21 cachetools==4.2.2 certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +cffi==1.14.6 +charset-normalizer==2.0.4 configparser==5.0.2 coreapi==2.3.3 coreschema==0.0.4 coverage==5.5 -cryptography==3.4.7 decorator==5.0.9 Django==2.2.24 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.7.0 -django-debug-toolbar==3.2.1 +django-cors-headers==3.8.0 +django-debug-toolbar==3.2.2 django-dynamic-fixture==3.1.1 django-filter==2.4.0 django-minio-storage==0.3.10 @@ -52,21 +51,21 @@ et-xmlfile==1.1.0 flake8==3.9.2 flake8-quotes==3.2.0 funcy==1.16 -google-api-core==1.29.0 -google-auth==1.30.1 -google-cloud-core==1.6.0 -google-cloud-storage==1.38.0 +google-api-core==1.31.1 +google-auth==1.34.0 +google-cloud-core==1.7.2 +google-cloud-storage==1.42.0 google-crc32c==1.1.2 -google-resumable-media==1.3.0 +google-resumable-media==1.3.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.0 -idna==2.10 +greenlet==1.1.1 +idna==3.2 inflection==0.5.1 itypes==1.2.0 Jinja2==3.0.1 jmespath==0.10.0 -jsonpath-ng==1.5.2 +jsonpath-ng==1.5.3 jsonschema==3.2.0 lxml==4.6.3 Markdown==3.3.4 @@ -74,36 +73,35 @@ MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 openpyxl==3.0.7 -packaging==20.9 +packaging==21.0 ply==3.11 prometheus-client==0.11.0 -protobuf==3.17.2 -psycopg2-binary==2.8.6 +protobuf==3.17.3 +psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 pycparser==2.20 pyflakes==2.3.1 -Pygments==2.9.0 -pyOpenSSL==20.0.1 +Pygments==2.10.0 pyparsing==2.4.7 -pyrsistent==0.17.3 -python-dateutil==2.8.1 -python-json-logger==2.0.1 +pyrsistent==0.18.0 +python-dateutil==2.8.2 +python-json-logger==2.0.2 pytz==2021.1 redis==3.5.3 -requests==2.25.1 +requests==2.26.0 rsa==4.7.2 -ruamel.yaml==0.17.7 -ruamel.yaml.clib==0.2.2 -s3transfer==0.4.2 -sentry-sdk==1.1.0 +ruamel.yaml==0.17.10 +ruamel.yaml.clib==0.2.6 +s3transfer==0.5.0 +sentry-sdk==1.3.1 six==1.16.0 -spavro==1.1.23 -SQLAlchemy==1.4.17 +spavro==1.1.24 +SQLAlchemy==1.4.22 sqlparse==0.4.1 tblib==1.7.0 toml==0.10.2 uritemplate==3.0.1 -urllib3==1.26.5 +urllib3==1.26.6 uWSGI==2.0.19.1 diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt index 3539c1c49..8ffcbad31 100644 --- a/aether-odk-module/conf/pip/requirements.txt +++ b/aether-odk-module/conf/pip/requirements.txt @@ -14,20 +14,19 @@ aether.sdk==1.3.8 autopep8==1.5.7 -boto3==1.17.86 -botocore==1.20.86 +boto3==1.18.21 +botocore==1.21.21 cachetools==4.2.2 certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +cffi==1.14.6 +charset-normalizer==2.0.4 configparser==5.0.2 coverage==5.5 -cryptography==3.4.7 Django==2.2.24 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.7.0 -django-debug-toolbar==3.2.1 +django-cors-headers==3.8.0 +django-debug-toolbar==3.2.2 django-minio-storage==0.3.10 django-postgrespool2==2.0.1 django-prometheus==2.1.0 @@ -41,16 +40,16 @@ flake8==3.9.2 flake8-quotes==3.2.0 FormEncode==1.3.1 funcy==1.16 -google-api-core==1.29.0 -google-auth==1.30.1 -google-cloud-core==1.6.0 -google-cloud-storage==1.38.0 +google-api-core==1.31.1 +google-auth==1.34.0 +google-cloud-core==1.7.2 +google-cloud-storage==1.42.0 google-crc32c==1.1.2 -google-resumable-media==1.3.0 +google-resumable-media==1.3.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.0 -idna==2.10 +greenlet==1.1.1 +idna==3.2 Jinja2==3.0.1 jmespath==0.10.0 linecache2==1.0.0 @@ -59,36 +58,35 @@ Markdown==3.3.4 MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 -packaging==20.9 +packaging==21.0 prometheus-client==0.11.0 -protobuf==3.17.2 -psycopg2-binary==2.8.6 +protobuf==3.17.3 +psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 pycparser==2.20 pyflakes==2.3.1 -Pygments==2.9.0 -pyOpenSSL==20.0.1 +Pygments==2.10.0 pyparsing==2.4.7 -python-dateutil==2.8.1 -python-json-logger==2.0.1 +python-dateutil==2.8.2 +python-json-logger==2.0.2 pytz==2021.1 pyxform==1.5.1 redis==3.5.3 -requests==2.25.1 +requests==2.26.0 rsa==4.7.2 -s3transfer==0.4.2 -sentry-sdk==1.1.0 +s3transfer==0.5.0 +sentry-sdk==1.3.1 six==1.16.0 -spavro==1.1.23 -SQLAlchemy==1.4.17 +spavro==1.1.24 +SQLAlchemy==1.4.22 sqlparse==0.4.1 tblib==1.7.0 toml==0.10.2 traceback2==1.4.0 unicodecsv==0.14.1 unittest2==1.1.0 -urllib3==1.26.5 +urllib3==1.26.6 uWSGI==2.0.19.1 xlrd==1.2.0 diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt index 5977a3334..79947e896 100644 --- a/aether-producer/conf/pip/requirements.txt +++ b/aether-producer/conf/pip/requirements.txt @@ -14,39 +14,35 @@ attrs==21.2.0 certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +charset-normalizer==2.0.4 click==8.0.1 confluent-kafka==1.7.0 -cryptography==3.4.7 flake8==3.9.2 flake8-quotes==3.2.0 Flask==2.0.1 -gevent==21.1.2 -greenlet==1.1.0 -idna==2.10 +gevent==21.8.0 +greenlet==1.1.1 +idna==3.2 iniconfig==1.1.1 itsdangerous==2.0.1 Jinja2==3.0.1 MarkupSafe==2.0.1 mccabe==0.6.1 -packaging==20.9 +packaging==21.0 pluggy==0.13.1 psycogreen==1.0.2 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.1 py==1.10.0 pycodestyle==2.7.0 -pycparser==2.20 pyflakes==2.3.1 -pyOpenSSL==20.0.1 pyparsing==2.4.7 pytest==6.2.4 -requests==2.25.1 +requests==2.26.0 six==1.16.0 -spavro==1.1.23 -SQLAlchemy==1.4.17 +spavro==1.1.24 +SQLAlchemy==1.4.22 toml==0.10.2 -urllib3==1.26.5 +urllib3==1.26.6 Werkzeug==2.0.1 zope.event==4.5.0 zope.interface==5.4.0 diff --git a/aether-ui/aether/ui/assets/css/base/_fonts.scss b/aether-ui/aether/ui/assets/css/base/_fonts.scss index 2a3373705..39e865bbe 100644 --- a/aether-ui/aether/ui/assets/css/base/_fonts.scss +++ b/aether-ui/aether/ui/assets/css/base/_fonts.scss @@ -23,5 +23,5 @@ @import url('https://fonts.googleapis.com/css?family=Fira+Mono'); /* Font Awesome 5 https://fontawesome.com/ */ -@import url('https://use.fontawesome.com/releases/v5.15.2/css/fontawesome.css'); -@import url('https://use.fontawesome.com/releases/v5.15.2/css/solid.css'); +@import url('https://use.fontawesome.com/releases/v5.15.4/css/fontawesome.css'); +@import url('https://use.fontawesome.com/releases/v5.15.4/css/solid.css'); diff --git a/aether-ui/aether/ui/assets/package.json b/aether-ui/aether/ui/assets/package.json index c0bf85e02..c100c2e99 100644 --- a/aether-ui/aether/ui/assets/package.json +++ b/aether-ui/aether/ui/assets/package.json @@ -20,35 +20,35 @@ "start": "node ./conf/webpack.server.js" }, "dependencies": { - "avsc": "~5.5.0", + "avsc": "~5.7.0", "bootstrap": "~4.6.0", "html5shiv": "~3.7.0", - "jquery": "~3.5.0", + "jquery": "~3.6.0", "moment": "~2.29.0", "popper.js": "~1.16.0", "react": "~16.14.0", "react-clipboard.js": "~2.0.16", "react-dom": "~16.14.0", - "react-intl": "~5.12.0", + "react-intl": "~5.20.0", "react-outside-click-handler": "~1.3.0", "react-redux": "~7.2.0", "react-router-dom": "~5.2.0", - "redux": "~4.0.0", + "redux": "~4.1.0", "redux-thunk": "~2.3.0", "uuid": "~8.3.0", "webpack-google-cloud-storage-plugin": "~0.9.0", "webpack-s3-plugin": "~1.0.3", - "whatwg-fetch": "~3.5.0" + "whatwg-fetch": "~3.6.0" }, "devDependencies": { - "@babel/core": "~7.12.0", - "@babel/plugin-proposal-class-properties": "~7.12.0", - "@babel/preset-env": "~7.12.0", - "@babel/preset-react": "~7.12.0", + "@babel/core": "~7.15.0", + "@babel/plugin-proposal-class-properties": "~7.14.0", + "@babel/preset-env": "~7.15.0", + "@babel/preset-react": "~7.14.0", "@hot-loader/react-dom": "~16.14.0", "babel-loader": "~8.2.0", "css-loader": "~5.0.0", - "eslint": "~7.19.0", + "eslint": "~7.32.0", "enzyme": "~3.11.0", "enzyme-adapter-react-16": "~1.15.0", "express": "~4.17.0", @@ -62,8 +62,8 @@ "sass-loader": "~10.1.0", "standard": "~16.0.0", "style-loader": "~2.0.0", - "stylelint": "~13.10.0", - "stylelint-config-standard": "~20.0.0", + "stylelint": "~13.13.0", + "stylelint-config-standard": "~22.0.0", "webpack": "~4.44.0", "webpack-bundle-tracker": "~0.4.3", "webpack-cli": "~3.3.0", diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt index 376bc0084..1c485e266 100644 --- a/aether-ui/conf/pip/requirements.txt +++ b/aether-ui/conf/pip/requirements.txt @@ -14,20 +14,19 @@ aether.sdk==1.3.8 autopep8==1.5.7 -boto3==1.17.86 -botocore==1.20.86 +boto3==1.18.21 +botocore==1.21.21 cachetools==4.2.2 certifi==2021.5.30 -cffi==1.14.5 -chardet==4.0.0 +cffi==1.14.6 +charset-normalizer==2.0.4 configparser==5.0.2 coverage==5.5 -cryptography==3.4.7 Django==2.2.24 django-cacheops==6.0 django-cleanup==5.2.0 -django-cors-headers==3.7.0 -django-debug-toolbar==3.2.1 +django-cors-headers==3.8.0 +django-debug-toolbar==3.2.2 django-minio-storage==0.3.10 django-model-utils==4.1.1 django-postgrespool2==2.0.1 @@ -42,46 +41,45 @@ drf-dynamic-fields==0.3.1 flake8==3.9.2 flake8-quotes==3.2.0 funcy==1.16 -google-api-core==1.29.0 -google-auth==1.30.1 -google-cloud-core==1.6.0 -google-cloud-storage==1.38.0 +google-api-core==1.31.1 +google-auth==1.34.0 +google-cloud-core==1.7.2 +google-cloud-storage==1.42.0 google-crc32c==1.1.2 -google-resumable-media==1.3.0 +google-resumable-media==1.3.3 googleapis-common-protos==1.53.0 gprof2dot==2021.2.21 -greenlet==1.1.0 -idna==2.10 +greenlet==1.1.1 +idna==3.2 Jinja2==3.0.1 jmespath==0.10.0 Markdown==3.3.4 MarkupSafe==2.0.1 mccabe==0.6.1 minio==6.0.2 -packaging==20.9 +packaging==21.0 prometheus-client==0.11.0 -protobuf==3.17.2 -psycopg2-binary==2.8.6 +protobuf==3.17.3 +psycopg2-binary==2.9.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 pycparser==2.20 pyflakes==2.3.1 -Pygments==2.9.0 -pyOpenSSL==20.0.1 +Pygments==2.10.0 pyparsing==2.4.7 -python-dateutil==2.8.1 -python-json-logger==2.0.1 +python-dateutil==2.8.2 +python-json-logger==2.0.2 pytz==2021.1 redis==3.5.3 -requests==2.25.1 +requests==2.26.0 rsa==4.7.2 -s3transfer==0.4.2 -sentry-sdk==1.1.0 +s3transfer==0.5.0 +sentry-sdk==1.3.1 six==1.16.0 -SQLAlchemy==1.4.17 +SQLAlchemy==1.4.22 sqlparse==0.4.1 tblib==1.7.0 toml==0.10.2 -urllib3==1.26.5 +urllib3==1.26.6 uWSGI==2.0.19.1 From 7e957ddb0dce4dbc17ab267d55b1139916321592 Mon Sep 17 00:00:00 2001 From: obdulia-losantos Date: Mon, 16 Aug 2021 15:01:51 +0200 Subject: [PATCH 2/3] feat: add endpoint to remove all project data --- .../aether/kernel/api/tests/test_views.py | 17 +++++++++++++++++ aether-kernel/aether/kernel/api/views.py | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/aether-kernel/aether/kernel/api/tests/test_views.py b/aether-kernel/aether/kernel/api/tests/test_views.py index fef7c54a3..6c75aae0f 100644 --- a/aether-kernel/aether/kernel/api/tests/test_views.py +++ b/aether-kernel/aether/kernel/api/tests/test_views.py @@ -601,6 +601,23 @@ def test_schema_validate_definition__errors(self): ) self.assertEqual(response.status_code, 400) + def test_project_delete_data(self): + self.assertEqual(reverse('project-erase_data', kwargs={'pk': 1}), + '/projects/1/delete-data/') + url = reverse('project-erase_data', kwargs={'pk': self.project.pk}) + + submissions = models.Submission.objects.filter(mappingset__project=self.project) + entities = models.Entity.objects.filter(project=self.project) + + self.assertGreater(submissions.count(), 0) + self.assertGreater(entities.count(), 0) + + response = self.client.patch(url) + self.assertEqual(response.status_code, 204) + + self.assertEqual(submissions.count(), 0) + self.assertEqual(entities.count(), 0) + def test_project__schemas_skeleton(self): self.assertEqual(reverse('project-skeleton', kwargs={'pk': 1}), '/projects/1/schemas-skeleton/') diff --git a/aether-kernel/aether/kernel/api/views.py b/aether-kernel/aether/kernel/api/views.py index 7a460ff66..e1e5f652e 100644 --- a/aether-kernel/aether/kernel/api/views.py +++ b/aether-kernel/aether/kernel/api/views.py @@ -55,6 +55,19 @@ class ProjectViewSet(MtViewSetMixin, FilteredMixin, ExtractMixin, viewsets.Model filter_class = filters.ProjectFilter search_fields = ('name',) + @action(detail=True, methods=['patch'], url_name='erase_data', url_path='delete-data') + def delete_data(self, request, pk=None, *args, **kwargs): + instance = self.get_object_or_404(pk=pk) + try: + instance.submissions.all().delete() + instance.entities.all().delete() + return Response(status=status.HTTP_204_NO_CONTENT) + except Exception as e: # pragma: no cover + return Response( + str(e), + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + @action(detail=True, methods=['get'], url_name='skeleton', url_path='schemas-skeleton') def schemas_skeleton(self, request, pk=None, *args, **kwargs): ''' From a80fc43fb24a3e07123ac6c77e10c0582e766e26 Mon Sep 17 00:00:00 2001 From: obdulia-losantos Date: Mon, 16 Aug 2021 16:27:24 +0200 Subject: [PATCH 3/3] fix: revert psycopg2-binary upgrade --- aether-kernel/conf/pip/requirements.txt | 2 +- aether-odk-module/conf/pip/requirements.txt | 2 +- aether-producer/conf/pip/requirements.txt | 2 +- aether-ui/conf/pip/requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aether-kernel/conf/pip/requirements.txt b/aether-kernel/conf/pip/requirements.txt index ddd13c0c2..047189b5c 100644 --- a/aether-kernel/conf/pip/requirements.txt +++ b/aether-kernel/conf/pip/requirements.txt @@ -77,7 +77,7 @@ packaging==21.0 ply==3.11 prometheus-client==0.11.0 protobuf==3.17.3 -psycopg2-binary==2.9.1 +psycopg2-binary==2.8.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 diff --git a/aether-odk-module/conf/pip/requirements.txt b/aether-odk-module/conf/pip/requirements.txt index 8ffcbad31..ff16e7e23 100644 --- a/aether-odk-module/conf/pip/requirements.txt +++ b/aether-odk-module/conf/pip/requirements.txt @@ -61,7 +61,7 @@ minio==6.0.2 packaging==21.0 prometheus-client==0.11.0 protobuf==3.17.3 -psycopg2-binary==2.9.1 +psycopg2-binary==2.8.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0 diff --git a/aether-producer/conf/pip/requirements.txt b/aether-producer/conf/pip/requirements.txt index 79947e896..dbb7fb510 100644 --- a/aether-producer/conf/pip/requirements.txt +++ b/aether-producer/conf/pip/requirements.txt @@ -31,7 +31,7 @@ mccabe==0.6.1 packaging==21.0 pluggy==0.13.1 psycogreen==1.0.2 -psycopg2-binary==2.9.1 +psycopg2-binary==2.8.6 py==1.10.0 pycodestyle==2.7.0 pyflakes==2.3.1 diff --git a/aether-ui/conf/pip/requirements.txt b/aether-ui/conf/pip/requirements.txt index 1c485e266..893162d39 100644 --- a/aether-ui/conf/pip/requirements.txt +++ b/aether-ui/conf/pip/requirements.txt @@ -60,7 +60,7 @@ minio==6.0.2 packaging==21.0 prometheus-client==0.11.0 protobuf==3.17.3 -psycopg2-binary==2.9.1 +psycopg2-binary==2.8.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycodestyle==2.7.0