From ad8086e141fc524685e01d81904992f4eb544ea3 Mon Sep 17 00:00:00 2001 From: Lucas Lavandeira <19612265+lucaslavandeira@users.noreply.github.com> Date: Thu, 27 Dec 2018 18:06:25 -0300 Subject: [PATCH] Muevo tests, y utils relacionados al admin a la lib --- .../apps/metadata/tests/utils_tests.py | 42 +--- series_tiempo_ar_api/apps/metadata/utils.py | 12 -- .../libs/custom_admins/admin.py | 2 +- .../samples/single_distribution.json | 199 ++++++++++++++++++ .../libs/custom_admins/tests.py | 45 ++++ .../libs/custom_admins/utils.py | 10 + 6 files changed, 257 insertions(+), 53 deletions(-) create mode 100644 series_tiempo_ar_api/libs/custom_admins/samples/single_distribution.json create mode 100644 series_tiempo_ar_api/libs/custom_admins/tests.py create mode 100644 series_tiempo_ar_api/libs/custom_admins/utils.py diff --git a/series_tiempo_ar_api/apps/metadata/tests/utils_tests.py b/series_tiempo_ar_api/apps/metadata/tests/utils_tests.py index ca145724..72bda019 100644 --- a/series_tiempo_ar_api/apps/metadata/tests/utils_tests.py +++ b/series_tiempo_ar_api/apps/metadata/tests/utils_tests.py @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- import os -from django.contrib.admin import ACTION_CHECKBOX_NAME -from django.contrib.auth.models import User -from django.core import urlresolvers from django.test import TestCase -from django_datajsonar.indexing.catalog_reader import index_catalog -from mock import mock -from series_tiempo_ar_api.apps.metadata.utils import resolve_catalog_id_aliases, delete_metadata +from series_tiempo_ar_api.apps.metadata.utils import resolve_catalog_id_aliases from series_tiempo_ar_api.apps.metadata.models import CatalogAlias -from django_datajsonar.models import Node, Field, ReadDataJsonTask, Distribution +from django_datajsonar.models import Node SAMPLES_DIR = os.path.join(os.path.dirname(__file__), 'samples') @@ -50,36 +45,3 @@ def test_one_alias_one_catalog_id(self): expected = ['one_catalog', 'two_catalog', 'third_catalog'] self.assertEqual(set(expected), set(ids)) - - -class DeleteMetadataTests(TestCase): - distribution_change = urlresolvers.reverse('admin:django_datajsonar_distribution_changelist') - field_change = urlresolvers.reverse('admin:django_datajsonar_field_changelist') - - def setUp(self): - self.catalog_id = "one" - node = Node.objects.create(catalog_url=os.path.join(SAMPLES_DIR, 'single_distribution.json'), - catalog_id=self.catalog_id, - indexable=True) - index_catalog(node, ReadDataJsonTask.objects.create(), whitelist=True) - user = User.objects.create(username='a', password='b', is_staff=True, is_superuser=True) - self.client.force_login(user) - - def test_delete_fields(self): - with mock.patch('series_tiempo_ar_api.apps.metadata.admin.delete_metadata') as fake_delete: - fields = Field.objects.all() - self.client.post(self.field_change, {'action': 'delete_model', - ACTION_CHECKBOX_NAME: [str(f.pk) for f in fields]}, - follow=True) - - self.assertEqual(set(fake_delete.call_args[0][0]), set(fields)) - - def test_delete_distribution(self): - with mock.patch('series_tiempo_ar_api.apps.metadata.admin.delete_metadata') as fake_delete: - distribution: Distribution = Distribution.objects.filter(dataset__catalog__identifier=self.catalog_id).first() - fields = set(distribution.field_set.all()) - self.client.post(self.distribution_change, - {'action': 'delete_model', - ACTION_CHECKBOX_NAME: [str(distribution.pk)]}, - follow=True) - self.assertEqual(set(fake_delete.call_args[0][0]), set(fields)) diff --git a/series_tiempo_ar_api/apps/metadata/utils.py b/series_tiempo_ar_api/apps/metadata/utils.py index a2b046ed..1d8373d4 100644 --- a/series_tiempo_ar_api/apps/metadata/utils.py +++ b/series_tiempo_ar_api/apps/metadata/utils.py @@ -2,13 +2,8 @@ import random from typing import Sequence, List -from django.db.models import QuerySet from django.utils import timezone -from elasticsearch_dsl import Search, Q -from django_datajsonar.models import Field -from series_tiempo_ar_api.apps.metadata import constants from series_tiempo_ar_api.apps.metadata.models import CatalogAlias -from series_tiempo_ar_api.libs.indexing.elastic import ElasticInstance def resolve_catalog_id_aliases(aliases: Sequence[str]) -> List[str]: @@ -27,12 +22,5 @@ def resolve_catalog_id_aliases(aliases: Sequence[str]) -> List[str]: return catalog_ids -def delete_metadata(fields: list): - es_instance = ElasticInstance.get() - - search = Search(using=es_instance, index=constants.METADATA_ALIAS) - return search.filter('terms', id=[field.identifier for field in fields]).delete() - - def get_random_index_name(): return f"metadata-{random.randrange(1000000)}-{int(timezone.now().timestamp())}" diff --git a/series_tiempo_ar_api/libs/custom_admins/admin.py b/series_tiempo_ar_api/libs/custom_admins/admin.py index 2449e734..bff738fb 100644 --- a/series_tiempo_ar_api/libs/custom_admins/admin.py +++ b/series_tiempo_ar_api/libs/custom_admins/admin.py @@ -3,7 +3,7 @@ from django_datajsonar.admin import FieldAdmin, DistributionAdmin from django_datajsonar.models import Field, Distribution -from series_tiempo_ar_api.apps.metadata.utils import delete_metadata +from .utils import delete_metadata from .tasks import reindex_distribution admin.site.unregister(Field) diff --git a/series_tiempo_ar_api/libs/custom_admins/samples/single_distribution.json b/series_tiempo_ar_api/libs/custom_admins/samples/single_distribution.json new file mode 100644 index 00000000..5992a977 --- /dev/null +++ b/series_tiempo_ar_api/libs/custom_admins/samples/single_distribution.json @@ -0,0 +1,199 @@ +{ + "publisher": { + "mbox": "datoseconomicos@mecon.gov.ar", + "name": "Subsecretaría de Programación Macroeconómica." + }, + "license": "Open Database License (ODbL) v1.0", + "description": "Catálogo de datos abiertos de la Subsecretaría de Programación Macroeconómica.", + "language": [ + "SPA" + ], + "superThemeTaxonomy": "http://datos.gob.ar/superThemeTaxonomy.json", + "issued": "2017-09-28", + "rights": "2017-09-28", + "modified": "2017-09-28", + "dataset": [ + { + "publisher": { + "mbox": "datoseconomicos@mecon.gov.ar", + "name": "Subsecretaría de Programación Macroeconómica." + }, + "landingPage": "http://www.minhacienda.gob.ar/secretarias/politica-economica/programacion-macroeconomica/", + "keyword": [ + "Información Económica Histórica", + "Actividad" + ], + "superTheme": [ + "ECON" + ], + "title": "Datos históricos de la industria siderúrgica", + "language": [ + "SPA" + ], + "issued": "2017-12-12", + "temporal": "1960-01-01/2016-12-01", + "source": "Cámara Argentina del Acero y Boletín Informativo de Techint", + "theme": [ + "actividad" + ], + "accrualPeriodicity": "R/P1M", + "spatial": "ARG", + "identifier": "359", + "license": "Creative Commons Attribution 4.0", + "contactPoint": { + "fn": "Subsecretaría de Programación Macroeconómica." + }, + "accessLevel": "ABIERTO", + "distribution": [ + { + "accessURL": "https://www.minhacienda.gob.ar/secretarias/politica-economica/programacion-macroeconomica/", + "scrapingFileSheet": "1.23 Historicos Siderurgia", + "description": "Datos históricos del volumen de producción de la industria siderúrgica. Datos Anuales", + "format": "CSV", + "dataset_identifier": "359", + "issued": "2017-12-12", + "title": "Datos históricos de la industria siderúrgica. Datos Anuales", + "fileName": "datos-historicos-de-la-industria-siderurgica-datos-anuales.csv", + "downloadURL": "http://infra.datos.gob.ar/catalog/sspm/dataset/359/distribution/359.1/download/datos-historicos-de-la-industria-siderurgica-datos-anuales.csv", + "field": [ + { + "distribution_identifier": "359.1", + "title": "indice_tiempo", + "dataset_identifier": "359", + "scrapingIdentifierCell": "A9", + "specialTypeDetail": "R/P1Y", + "specialType": "time_index", + "type": "date", + "id": "359.1_INDICE_TIEMPO__13", + "scrapingDataStartCell": "A10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de hierro primario arrabio", + "title": "hierro_primario_arrabio", + "dataset_identifier": "359", + "scrapingIdentifierCell": "B9", + "units": "Unidades", + "type": "number", + "id": "359.1_HIERRO_PRIBIO__23", + "scrapingDataStartCell": "B10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de hierro primario hierro esponja", + "title": "hierro_primario_hierro_esponja", + "dataset_identifier": "359", + "scrapingIdentifierCell": "C9", + "units": "Unidades", + "type": "number", + "id": "359.1_HIERRO_PRINJA__30", + "scrapingDataStartCell": "C10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de hierro primario total", + "title": "hierro_primario_total", + "dataset_identifier": "359", + "scrapingIdentifierCell": "D9", + "units": "Unidades", + "type": "number", + "id": "359.1_HIERRO_PRITAL__21", + "scrapingDataStartCell": "D10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de acero crudo", + "title": "acero_crudo", + "dataset_identifier": "359", + "scrapingIdentifierCell": "F9", + "units": "Unidades", + "type": "number", + "id": "359.1_ACERO_CRUDUDO__11", + "scrapingDataStartCell": "F10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de laminados no planos en caliente", + "title": "laminados_no_planos_en_caliente", + "dataset_identifier": "359", + "scrapingIdentifierCell": "H9", + "units": "Unidades", + "type": "number", + "id": "359.1_LAMINADOS_NTE__31", + "scrapingDataStartCell": "H10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de laminados planos en caliente", + "title": "laminados_planos_en_caliente", + "dataset_identifier": "359", + "scrapingIdentifierCell": "I9", + "units": "Unidades", + "type": "number", + "id": "359.1_LAMINADOS_NTE__28", + "scrapingDataStartCell": "I10" + }, + { + "distribution_identifier": "359.1", + "description": "Volúmen de produción de laminados planos en frio", + "title": "laminados_planos_en_frio", + "dataset_identifier": "359", + "scrapingIdentifierCell": "J9", + "units": "Unidades", + "type": "number", + "id": "359.1_LAMINADOS_RIO__24", + "scrapingDataStartCell": "J10" + } + ], + "identifier": "359.1", + "scrapingFileURL": "https://www.economia.gob.ar/download/infoeco/actividad_ied.xlsx" + } + ] + } + ], + "identifier": "sspm", + "spatial": "ARG", + "title": "Datos Programación Macroeconómica", + "themeTaxonomy": [ + { + "label": "Actividad", + "id": "actividad", + "description": "Series de actividad: cuentas nacionales, consumo, producción, etc." + }, + { + "label": "Empleo e Ingresos", + "id": "empleo", + "description": "Series de empleo e ingresos: mercado laboral, salarios, etc." + }, + { + "label": "Sector Externo", + "id": "externo", + "description": "Series externas: Balance de Pagos, Intercambio Comercial, etc." + }, + { + "label": "Dinero y Bancos", + "id": "moneda", + "description": "Series monetarias y bancarias" + }, + { + "label": "Precios", + "id": "precios", + "description": "Series de precios" + }, + { + "label": "Economía Internacional", + "id": "internacional", + "description": "Series económicas mundiales" + }, + { + "label": "Finanzas Públicas", + "id": "fiscal", + "description": "Series de recaudación, ingresos, gastos, etc." + }, + { + "label": "Mercados Financieros", + "id": "finanzas", + "description": "Series financieras y de mercado de capitales" + } + ] +} \ No newline at end of file diff --git a/series_tiempo_ar_api/libs/custom_admins/tests.py b/series_tiempo_ar_api/libs/custom_admins/tests.py new file mode 100644 index 00000000..3875c7ae --- /dev/null +++ b/series_tiempo_ar_api/libs/custom_admins/tests.py @@ -0,0 +1,45 @@ +import os + +import mock +from django.contrib.admin import ACTION_CHECKBOX_NAME +from django.contrib.auth.models import User +from django.core import urlresolvers +from django.test import TestCase +from django_datajsonar.models import Node, ReadDataJsonTask, Field, Distribution + +from django_datajsonar.indexing.catalog_reader import index_catalog + +SAMPLES_DIR = os.path.join(os.path.dirname(__file__), 'samples') + + +class DeleteMetadataTests(TestCase): + distribution_change = urlresolvers.reverse('admin:django_datajsonar_distribution_changelist') + field_change = urlresolvers.reverse('admin:django_datajsonar_field_changelist') + + def setUp(self): + self.catalog_id = "one" + node = Node.objects.create(catalog_url=os.path.join(SAMPLES_DIR, 'single_distribution.json'), + catalog_id=self.catalog_id, + indexable=True) + index_catalog(node, ReadDataJsonTask.objects.create(), whitelist=True) + user = User.objects.create(username='a', password='b', is_staff=True, is_superuser=True) + self.client.force_login(user) + + def test_delete_fields(self): + with mock.patch('series_tiempo_ar_api.libs.custom_admins.admin.delete_metadata') as fake_delete: + fields = Field.objects.all() + self.client.post(self.field_change, {'action': 'delete_model', + ACTION_CHECKBOX_NAME: [str(f.pk) for f in fields]}, + follow=True) + + self.assertEqual(set(fake_delete.call_args[0][0]), set(fields)) + + def test_delete_distribution(self): + with mock.patch('series_tiempo_ar_api.libs.custom_admins.admin.delete_metadata') as fake_delete: + distribution: Distribution = Distribution.objects.filter(dataset__catalog__identifier=self.catalog_id).first() + fields = set(distribution.field_set.all()) + self.client.post(self.distribution_change, + {'action': 'delete_model', + ACTION_CHECKBOX_NAME: [str(distribution.pk)]}, + follow=True) + self.assertEqual(set(fake_delete.call_args[0][0]), set(fields)) diff --git a/series_tiempo_ar_api/libs/custom_admins/utils.py b/series_tiempo_ar_api/libs/custom_admins/utils.py new file mode 100644 index 00000000..b10d26f1 --- /dev/null +++ b/series_tiempo_ar_api/libs/custom_admins/utils.py @@ -0,0 +1,10 @@ +from elasticsearch_dsl import Search, Q +from series_tiempo_ar_api.apps.metadata import constants +from series_tiempo_ar_api.libs.indexing.elastic import ElasticInstance + + +def delete_metadata(fields: list): + es_instance = ElasticInstance.get() + + search = Search(using=es_instance, index=constants.METADATA_ALIAS) + return search.filter('terms', id=[field.identifier for field in fields]).delete()