From 969367bfbbaff583765d7e2b5f908a58f58527a8 Mon Sep 17 00:00:00 2001 From: Lucas Lavandeira <19612265+lucaslavandeira@users.noreply.github.com> Date: Thu, 29 Nov 2018 14:48:01 -0300 Subject: [PATCH] =?UTF-8?q?Refactor=20de=20modelos=20singleton=20de=20conf?= =?UTF-8?q?iguraci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Se separa el comportamiento del comando ejecutado por IndexingTaskCron (comando hardcodeado por variable de entorno) en dos separadas, una para la indexación de metadatos y la otra para el importado de analytics de api mgmt. El horario de ejecución diario de estas tareas pasa a estar configurado a través de modelos singleton: ImportConfig de analytics para el importado, y MetadataConfig para la indexación de metadatos. TaskCron pasa a ser un modelo de uso interno por la aplicación. --- conf/settings/base.py | 5 +-- conf/settings/production.py | 3 ++ series_tiempo_ar_api/apps/analytics/admin.py | 6 ++-- .../migrations/0010_importconfig_time.py | 21 ++++++++++++ series_tiempo_ar_api/apps/analytics/models.py | 13 ++++++++ .../migrations/0008_auto_20181129_1247.py | 25 ++++++++++++++ series_tiempo_ar_api/apps/management/admin.py | 6 ++-- .../migrations/0001_auto_20181129_1247.py | 28 ++++++++++++++++ .../apps/management/migrations/reset_crons.py | 33 +++++++++++++++++++ .../apps/management/models.py | 12 +++---- .../apps/management/tests/cron_tests.py | 11 ++++--- series_tiempo_ar_api/apps/metadata/admin.py | 8 ++++- .../migrations/0003_metadataconfig.py | 26 +++++++++++++++ series_tiempo_ar_api/apps/metadata/models.py | 17 ++++++++++ series_tiempo_ar_api/libs/singleton_admin.py | 6 ++++ 15 files changed, 200 insertions(+), 20 deletions(-) create mode 100644 series_tiempo_ar_api/apps/analytics/migrations/0010_importconfig_time.py create mode 100644 series_tiempo_ar_api/apps/dump/migrations/0008_auto_20181129_1247.py create mode 100644 series_tiempo_ar_api/apps/management/migrations/0001_auto_20181129_1247.py create mode 100644 series_tiempo_ar_api/apps/management/migrations/reset_crons.py create mode 100644 series_tiempo_ar_api/apps/metadata/migrations/0003_metadataconfig.py create mode 100644 series_tiempo_ar_api/libs/singleton_admin.py diff --git a/conf/settings/base.py b/conf/settings/base.py index 1968788e..dc449bf6 100644 --- a/conf/settings/base.py +++ b/conf/settings/base.py @@ -303,8 +303,9 @@ def export_vars(_): ENV_TYPE = env('ENV_TYPE', default='') -# Tarea a ser croneada para indexación -READ_DATAJSON_SHELL_CMD = env('READ_DATAJSON_BIN_PATH', default='') +# Tarea a ser croneada para indexación. Defaults para uso local, en producción se deben setear estas variables! +IMPORT_ANALYTICS_SCRIPT_PATH = env('IMPORT_ANALYTICS_CMD_PATH', default='/bin/true import_analytics') +INDEX_METADATA_SCRIPT_PATH = env('INDEX_METADATA_CMD_PATH', default='/bin/true index_metadata') PROTECTED_MEDIA_DIR = env('PROTECTED_MEDIA_DIR', default=ROOT_DIR('protected')) ANALYTICS_CSV_FILENAME = 'analytics.csv' diff --git a/conf/settings/production.py b/conf/settings/production.py index 4fac13ef..087359e8 100644 --- a/conf/settings/production.py +++ b/conf/settings/production.py @@ -50,3 +50,6 @@ MINIO_SERVE_FILES_URL = '/series/files/' + +IMPORT_ANALYTICS_SCRIPT_PATH = env('IMPORT_ANALYTICS_CMD_PATH') +INDEX_METADATA_SCRIPT_PATH = env('INDEX_METADATA_CMD_PATH') diff --git a/series_tiempo_ar_api/apps/analytics/admin.py b/series_tiempo_ar_api/apps/analytics/admin.py index 10ebcb3f..e3703b7f 100644 --- a/series_tiempo_ar_api/apps/analytics/admin.py +++ b/series_tiempo_ar_api/apps/analytics/admin.py @@ -6,6 +6,7 @@ from django.db import connection from solo.admin import SingletonModelAdmin +from series_tiempo_ar_api.libs.singleton_admin import SingletonAdmin from .models import Query, ImportConfig, AnalyticsImportTask from .tasks import import_analytics_from_api_mgmt @@ -51,9 +52,8 @@ def get_readonly_fields(self, request, obj=None): return [field.name for field in self.opts.local_fields] -class ImportConfigAdmin(SingletonModelAdmin): - # django-des overridea el change_form_template de la clase padre(!), volvemos al default de django - change_form_template = 'admin/change_form.html' +class ImportConfigAdmin(SingletonAdmin): + pass class ImportTaskAdmin(admin.ModelAdmin): diff --git a/series_tiempo_ar_api/apps/analytics/migrations/0010_importconfig_time.py b/series_tiempo_ar_api/apps/analytics/migrations/0010_importconfig_time.py new file mode 100644 index 00000000..7c9ef28f --- /dev/null +++ b/series_tiempo_ar_api/apps/analytics/migrations/0010_importconfig_time.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2018-11-29 15:47 +from __future__ import unicode_literals + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0009_importconfig_last_cursor'), + ] + + operations = [ + migrations.AddField( + model_name='importconfig', + name='time', + field=models.TimeField(default=datetime.time(0, 0), help_text='Los segundos serán ignorados'), + ), + ] diff --git a/series_tiempo_ar_api/apps/analytics/models.py b/series_tiempo_ar_api/apps/analytics/models.py index c7d51498..90a62ecd 100644 --- a/series_tiempo_ar_api/apps/analytics/models.py +++ b/series_tiempo_ar_api/apps/analytics/models.py @@ -1,11 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import datetime + import requests +from django.conf import settings from django.db import models from django.core.exceptions import ValidationError from solo.models import SingletonModel +from series_tiempo_ar_api.apps.management.models import TaskCron + class Query(models.Model): """Registro de queries exitosas, guardadas con el propósito de analytics""" @@ -27,9 +32,12 @@ def __unicode__(self): class ImportConfig(SingletonModel): + SCRIPT_PATH = settings.IMPORT_ANALYTICS_SCRIPT_PATH + endpoint = models.URLField() token = models.CharField(max_length=64) kong_api_id = models.CharField(max_length=64) + time = models.TimeField(help_text='Los segundos serán ignorados', default=datetime.time(hour=0, minute=0)) last_cursor = models.CharField(max_length=64, blank=True) @@ -58,6 +66,11 @@ def get_authorization_header(self): """Devuelve el header de auth formateado para usar en la libreria de requests""" return {'Authorization': 'Token {}'.format(self.token)} + def save(self, *args, **kwargs): + super(ImportConfig, self).save(*args, **kwargs) + TaskCron.objects.update_or_create(task_script_path=self.SCRIPT_PATH, + defaults={'time': self.time}) + class AnalyticsImportTask(models.Model): diff --git a/series_tiempo_ar_api/apps/dump/migrations/0008_auto_20181129_1247.py b/series_tiempo_ar_api/apps/dump/migrations/0008_auto_20181129_1247.py new file mode 100644 index 00000000..9425c84d --- /dev/null +++ b/series_tiempo_ar_api/apps/dump/migrations/0008_auto_20181129_1247.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2018-11-29 15:47 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dump', '0007_auto_20181102_1542'), + ] + + operations = [ + migrations.AlterField( + model_name='dumpfile', + name='file_type', + field=models.CharField(choices=[('csv', 'CSV'), ('xlsx', 'XLSX'), ('zip', 'ZIP'), ('sqlite', 'SQL'), ('dta', 'DTA')], default='csv', max_length=12), + ), + migrations.AlterField( + model_name='generatedumptask', + name='file_type', + field=models.CharField(choices=[('csv', 'CSV'), ('xlsx', 'XLSX'), ('sql', 'SQL'), ('dta', 'DTA')], default='CSV', max_length=12), + ), + ] diff --git a/series_tiempo_ar_api/apps/management/admin.py b/series_tiempo_ar_api/apps/management/admin.py index 0f3eb4fc..a9eb58d1 100644 --- a/series_tiempo_ar_api/apps/management/admin.py +++ b/series_tiempo_ar_api/apps/management/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin from .tasks import read_datajson -from .models import IndexingTaskCron, ReadDataJsonTask +from .models import TaskCron, ReadDataJsonTask class NodeAdmin(admin.ModelAdmin): @@ -43,7 +43,7 @@ def get_actions(self, request): def delete_model(self, _, queryset): # Actualizo los crons del sistema para reflejar el cambio de modelos queryset.delete() - IndexingTaskCron.update_crontab() + TaskCron.update_crontab() class DataJsonAdmin(admin.ModelAdmin): @@ -65,5 +65,5 @@ def save_model(self, request, obj, form, change): read_datajson.delay(obj, force=force) # Ejecuta indexación -admin.site.register(IndexingTaskCron, IndexingTaskAdmin) +admin.site.register(TaskCron, IndexingTaskAdmin) admin.site.register(ReadDataJsonTask, DataJsonAdmin) diff --git a/series_tiempo_ar_api/apps/management/migrations/0001_auto_20181129_1247.py b/series_tiempo_ar_api/apps/management/migrations/0001_auto_20181129_1247.py new file mode 100644 index 00000000..70bf1b95 --- /dev/null +++ b/series_tiempo_ar_api/apps/management/migrations/0001_auto_20181129_1247.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2018-11-29 15:47 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('management', 'reset_crons'), + ] + + operations = [ + migrations.CreateModel( + name='TaskCron', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time', models.TimeField(help_text='Los segundos serán ignorados')), + ('enabled', models.BooleanField(default=True)), + ('weekdays_only', models.BooleanField(default=False)), + ('task_script_path', models.CharField(default=None, max_length=255)), + ], + ), + migrations.DeleteModel( + name='IndexingTaskCron', + ), + ] diff --git a/series_tiempo_ar_api/apps/management/migrations/reset_crons.py b/series_tiempo_ar_api/apps/management/migrations/reset_crons.py new file mode 100644 index 00000000..e0a695a3 --- /dev/null +++ b/series_tiempo_ar_api/apps/management/migrations/reset_crons.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +"""Esta migración no va a funcionar a futuro si pasamos a dejar usar el storage de minio +Debería borrarse en el caso que el storage no sea más minio! +""" +from __future__ import unicode_literals + +import os + +from django.conf import settings +from django.db import migrations +from django.core.files import File +from minio_storage.errors import MinIOError +from minio_storage.storage import MinioMediaStorage + +from django_datajsonar.models import Distribution + + +def migrate_files(apps, schema_editor): + IndexingTaskCron = apps.get_model('django_datajsonar', 'Metadata') + db_alias = schema_editor.connection.alias + + IndexingTaskCron.objects.using(db_alias).all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('management', 'migrate_files_to_minio'), + ] + + operations = [ + migrations.RunPython(migrate_files, reverse_code=lambda x, y: None) + ] diff --git a/series_tiempo_ar_api/apps/management/models.py b/series_tiempo_ar_api/apps/management/models.py index da5cf09a..ddf328e1 100644 --- a/series_tiempo_ar_api/apps/management/models.py +++ b/series_tiempo_ar_api/apps/management/models.py @@ -13,20 +13,21 @@ from .indicator_names import IndicatorNamesMixin -class IndexingTaskCron(models.Model): +class TaskCron(models.Model): cron_client = CronTab(user=getpass.getuser()) time = models.TimeField(help_text='Los segundos serán ignorados') enabled = models.BooleanField(default=True) weekdays_only = models.BooleanField(default=False) + task_script_path = models.CharField(max_length=255, default=None) def save(self, force_insert=False, force_update=False, using=None, update_fields=None): - super(IndexingTaskCron, self).save(force_insert, force_update, using, update_fields) + super(TaskCron, self).save(force_insert, force_update, using, update_fields) self.update_crontab() def delete(self, using=None, keep_parents=False): - super(IndexingTaskCron, self).delete(using, keep_parents) + super(TaskCron, self).delete(using, keep_parents) self.update_crontab() def __unicode__(self): @@ -35,16 +36,15 @@ def __unicode__(self): @classmethod def update_crontab(cls): """Limpia la crontab y la regenera a partir de los modelos de IndexingTaskCron guardados""" - command = settings.READ_DATAJSON_SHELL_CMD or 'true' cron = cls.cron_client job_id = strings.CRONTAB_COMMENT for job in cron.find_comment(job_id): job.delete() - tasks = IndexingTaskCron.objects.filter(enabled=True) + tasks = TaskCron.objects.filter(enabled=True) for task in tasks: - job = cron.new(command=command, comment=job_id) + job = cron.new(command=task.task_script_path, comment=job_id) job.minute.on(task.time.minute) job.hour.on(task.time.hour) diff --git a/series_tiempo_ar_api/apps/management/tests/cron_tests.py b/series_tiempo_ar_api/apps/management/tests/cron_tests.py index ec41d557..181f909f 100644 --- a/series_tiempo_ar_api/apps/management/tests/cron_tests.py +++ b/series_tiempo_ar_api/apps/management/tests/cron_tests.py @@ -4,14 +4,15 @@ import mock from django.test import TestCase -from series_tiempo_ar_api.apps.management.models import IndexingTaskCron +from series_tiempo_ar_api.apps.management.models import TaskCron class CronTests(TestCase): + script = '/bin/true' def test_cron_added(self): mock_write = mock.Mock(return_value=None) - cron = IndexingTaskCron(time='00:00:00') + cron = TaskCron(time='00:00:00', task_script_path=self.script) cron.cron_client.write = mock_write cron.save() @@ -20,7 +21,7 @@ def test_cron_added(self): def test_cron_removed(self): mock_write = mock.Mock(return_value=None) - cron = IndexingTaskCron(time='00:00:00') + cron = TaskCron(time='00:00:00', task_script_path=self.script) cron.cron_client.write = mock_write cron.save() @@ -34,7 +35,7 @@ def test_cron_scheduled_correctly(self): minute = 0 second = 0 mock_write = mock.Mock(return_value=None) - cron = IndexingTaskCron(time=time(hour=hour, minute=minute, second=second)) + cron = TaskCron(time=time(hour=hour, minute=minute, second=second), task_script_path=self.script) cron.cron_client.write = mock_write cron.save() @@ -47,7 +48,7 @@ def test_cron_scheduled_correctly(self): def test_cron_weekdays(self): mock_write = mock.Mock(return_value=None) - cron = IndexingTaskCron(time='00:00:00', weekdays_only=True) + cron = TaskCron(time='00:00:00', weekdays_only=True, task_script_path=self.script) cron.cron_client.write = mock_write cron.save() diff --git a/series_tiempo_ar_api/apps/metadata/admin.py b/series_tiempo_ar_api/apps/metadata/admin.py index 883baa55..22ced8e3 100644 --- a/series_tiempo_ar_api/apps/metadata/admin.py +++ b/series_tiempo_ar_api/apps/metadata/admin.py @@ -6,7 +6,8 @@ from django_datajsonar.admin import FieldAdmin, DistributionAdmin from django_datajsonar.models import Field, Distribution -from .models import IndexMetadataTask, CatalogAlias, Synonym +from series_tiempo_ar_api.libs.singleton_admin import SingletonAdmin +from .models import IndexMetadataTask, CatalogAlias, Synonym, MetadataConfig from .indexer.metadata_indexer import run_metadata_indexer from .utils import delete_metadata @@ -81,3 +82,8 @@ def delete_model(self, _, queryset): fields = Field.objects.filter(distribution__identifier__in=queryset.values_list('identifier', flat=True)) delete_metadata(list(fields)) queryset.delete() + + +@admin.register(MetadataConfig) +class MetadataConfigAdmin(SingletonAdmin): + pass diff --git a/series_tiempo_ar_api/apps/metadata/migrations/0003_metadataconfig.py b/series_tiempo_ar_api/apps/metadata/migrations/0003_metadataconfig.py new file mode 100644 index 00000000..2d8c6aa6 --- /dev/null +++ b/series_tiempo_ar_api/apps/metadata/migrations/0003_metadataconfig.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2018-11-29 17:10 +from __future__ import unicode_literals + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('metadata', '0002_auto_20180813_1510'), + ] + + operations = [ + migrations.CreateModel( + name='MetadataConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time', models.TimeField(default=datetime.time(0, 0), help_text='Los segundos serán ignorados')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/series_tiempo_ar_api/apps/metadata/models.py b/series_tiempo_ar_api/apps/metadata/models.py index a3672abf..2c0947b3 100644 --- a/series_tiempo_ar_api/apps/metadata/models.py +++ b/series_tiempo_ar_api/apps/metadata/models.py @@ -1,8 +1,14 @@ #! coding: utf-8 +import datetime from typing import Sequence, List + +from django.conf import settings from django.core.exceptions import ValidationError from django.db import models from django_datajsonar.models import AbstractTask, Node +from solo.models import SingletonModel + +from series_tiempo_ar_api.apps.management.models import TaskCron class IndexMetadataTask(AbstractTask): @@ -31,3 +37,14 @@ class Synonym(models.Model): terms = models.TextField( help_text='Lista de términos similares, separados por coma, sin espacios ni mayúsculas.' ' Ejemplo "ipc,inflacion"', unique=True) + + +class MetadataConfig(SingletonModel): + SCRIPT_PATH = settings.INDEX_METADATA_SCRIPT_PATH + + time = models.TimeField(help_text='Los segundos serán ignorados', default=datetime.time(hour=0, minute=0)) + + def save(self, *args, **kwargs): + super(MetadataConfig, self).save(*args, **kwargs) + TaskCron.objects.update_or_create(task_script_path=self.SCRIPT_PATH, + defaults={'time': self.time}) diff --git a/series_tiempo_ar_api/libs/singleton_admin.py b/series_tiempo_ar_api/libs/singleton_admin.py new file mode 100644 index 00000000..601ca80b --- /dev/null +++ b/series_tiempo_ar_api/libs/singleton_admin.py @@ -0,0 +1,6 @@ +from solo.admin import SingletonModelAdmin + + +class SingletonAdmin(SingletonModelAdmin): + # django-des overridea el change_form_template de la clase padre(!), volvemos al default de django + change_form_template = 'admin/change_form.html'