Merge pull request #425 from datosgobar/424-config-models

Refactor de modelos singleton de configuración

conf/settings/base.py

@@ -303,8 +303,9 @@ def export_vars(_):
 
 ENV_TYPE = env('ENV_TYPE', default='')
 
-# Tarea a ser croneada para indexación
-READ_DATAJSON_SHELL_CMD = env('READ_DATAJSON_BIN_PATH', default='')
+# Tarea a ser croneada para indexación. Defaults para uso local, en producción se deben setear estas variables!
+IMPORT_ANALYTICS_SCRIPT_PATH = env('IMPORT_ANALYTICS_CMD_PATH', default='/bin/true import_analytics')
+INDEX_METADATA_SCRIPT_PATH = env('INDEX_METADATA_CMD_PATH', default='/bin/true index_metadata')
 
 PROTECTED_MEDIA_DIR = env('PROTECTED_MEDIA_DIR', default=ROOT_DIR('protected'))
 ANALYTICS_CSV_FILENAME = 'analytics.csv'

conf/settings/production.py

@@ -50,3 +50,6 @@
 
 
 MINIO_SERVE_FILES_URL = '/series/files/'
+
+IMPORT_ANALYTICS_SCRIPT_PATH = env('IMPORT_ANALYTICS_CMD_PATH')
+INDEX_METADATA_SCRIPT_PATH = env('INDEX_METADATA_CMD_PATH')

series_tiempo_ar_api/apps/analytics/admin.py

@@ -6,6 +6,7 @@
 from django.db import connection
 from solo.admin import SingletonModelAdmin
 
+from series_tiempo_ar_api.libs.singleton_admin import SingletonAdmin
 from .models import Query, ImportConfig, AnalyticsImportTask
 from .tasks import import_analytics_from_api_mgmt
 
@@ -51,9 +52,8 @@ def get_readonly_fields(self, request, obj=None):
         return [field.name for field in self.opts.local_fields]
 
 
-class ImportConfigAdmin(SingletonModelAdmin):
-    # django-des overridea el change_form_template de la clase padre(!), volvemos al default de django
-    change_form_template = 'admin/change_form.html'
+class ImportConfigAdmin(SingletonAdmin):
+    pass
 
 
 class ImportTaskAdmin(admin.ModelAdmin):

series_tiempo_ar_api/apps/analytics/migrations/0010_importconfig_time.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-29 15:47
+from __future__ import unicode_literals
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0009_importconfig_last_cursor'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='importconfig',
+            name='time',
+            field=models.TimeField(default=datetime.time(0, 0), help_text='Los segundos serán ignorados'),
+        ),
+    ]

series_tiempo_ar_api/apps/analytics/models.py

@@ -1,11 +1,16 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import datetime
+
 import requests
+from django.conf import settings
 from django.db import models
 from django.core.exceptions import ValidationError
 from solo.models import SingletonModel
 
+from series_tiempo_ar_api.apps.management.models import TaskCron
+
 
 class Query(models.Model):
     """Registro de queries exitosas, guardadas con el propósito de analytics"""
@@ -27,9 +32,12 @@ def __unicode__(self):
 
 
 class ImportConfig(SingletonModel):
+    SCRIPT_PATH = settings.IMPORT_ANALYTICS_SCRIPT_PATH
+
     endpoint = models.URLField()
     token = models.CharField(max_length=64)
     kong_api_id = models.CharField(max_length=64)
+    time = models.TimeField(help_text='Los segundos serán ignorados', default=datetime.time(hour=0, minute=0))
 
     last_cursor = models.CharField(max_length=64, blank=True)
 
@@ -58,6 +66,11 @@ def get_authorization_header(self):
         """Devuelve el header de auth formateado para usar en la libreria de requests"""
         return {'Authorization': 'Token {}'.format(self.token)}
 
+    def save(self, *args, **kwargs):
+        super(ImportConfig, self).save(*args, **kwargs)
+        TaskCron.objects.update_or_create(task_script_path=self.SCRIPT_PATH,
+                                          defaults={'time': self.time})
+
 
 class AnalyticsImportTask(models.Model):
 

series_tiempo_ar_api/apps/dump/migrations/0008_auto_20181129_1247.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-29 15:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dump', '0007_auto_20181102_1542'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='dumpfile',
+            name='file_type',
+            field=models.CharField(choices=[('csv', 'CSV'), ('xlsx', 'XLSX'), ('zip', 'ZIP'), ('sqlite', 'SQL'), ('dta', 'DTA')], default='csv', max_length=12),
+        ),
+        migrations.AlterField(
+            model_name='generatedumptask',
+            name='file_type',
+            field=models.CharField(choices=[('csv', 'CSV'), ('xlsx', 'XLSX'), ('sql', 'SQL'), ('dta', 'DTA')], default='CSV', max_length=12),
+        ),
+    ]

series_tiempo_ar_api/apps/management/admin.py

@@ -3,7 +3,7 @@
 
 from django.contrib import admin
 from .tasks import read_datajson
-from .models import IndexingTaskCron, ReadDataJsonTask
+from .models import TaskCron, ReadDataJsonTask
 
 
 class NodeAdmin(admin.ModelAdmin):
@@ -43,7 +43,7 @@ def get_actions(self, request):
     def delete_model(self, _, queryset):
         # Actualizo los crons del sistema para reflejar el cambio de modelos
         queryset.delete()
-        IndexingTaskCron.update_crontab()
+        TaskCron.update_crontab()
 
 
 class DataJsonAdmin(admin.ModelAdmin):
@@ -65,5 +65,5 @@ def save_model(self, request, obj, form, change):
         read_datajson.delay(obj, force=force)  # Ejecuta indexación
 
 
-admin.site.register(IndexingTaskCron, IndexingTaskAdmin)
+admin.site.register(TaskCron, IndexingTaskAdmin)
 admin.site.register(ReadDataJsonTask, DataJsonAdmin)

series_tiempo_ar_api/apps/management/migrations/0001_auto_20181129_1247.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-29 15:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', 'reset_crons'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TaskCron',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('time', models.TimeField(help_text='Los segundos serán ignorados')),
+                ('enabled', models.BooleanField(default=True)),
+                ('weekdays_only', models.BooleanField(default=False)),
+                ('task_script_path', models.CharField(default=None, max_length=255)),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='IndexingTaskCron',
+        ),
+    ]

series_tiempo_ar_api/apps/management/migrations/reset_crons.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""Esta migración no va a funcionar a futuro si pasamos a dejar usar el storage de minio
+Debería borrarse en el caso que el storage no sea más minio!
+"""
+from __future__ import unicode_literals
+
+import os
+
+from django.conf import settings
+from django.db import migrations
+from django.core.files import File
+from minio_storage.errors import MinIOError
+from minio_storage.storage import MinioMediaStorage
+
+from django_datajsonar.models import Distribution
+
+
+def migrate_files(apps, schema_editor):
+    IndexingTaskCron = apps.get_model('django_datajsonar', 'Metadata')
+    db_alias = schema_editor.connection.alias
+
+    IndexingTaskCron.objects.using(db_alias).all().delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('management', 'migrate_files_to_minio'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_files, reverse_code=lambda x, y: None)
+    ]

series_tiempo_ar_api/apps/management/models.py

@@ -13,20 +13,21 @@
 from .indicator_names import IndicatorNamesMixin
 
 
-class IndexingTaskCron(models.Model):
+class TaskCron(models.Model):
     cron_client = CronTab(user=getpass.getuser())
 
     time = models.TimeField(help_text='Los segundos serán ignorados')
     enabled = models.BooleanField(default=True)
     weekdays_only = models.BooleanField(default=False)
+    task_script_path = models.CharField(max_length=255, default=None)
 
     def save(self, force_insert=False, force_update=False, using=None,
              update_fields=None):
-        super(IndexingTaskCron, self).save(force_insert, force_update, using, update_fields)
+        super(TaskCron, self).save(force_insert, force_update, using, update_fields)
         self.update_crontab()
 
     def delete(self, using=None, keep_parents=False):
-        super(IndexingTaskCron, self).delete(using, keep_parents)
+        super(TaskCron, self).delete(using, keep_parents)
         self.update_crontab()
 
     def __unicode__(self):
@@ -35,16 +36,15 @@ def __unicode__(self):
     @classmethod
     def update_crontab(cls):
         """Limpia la crontab y la regenera a partir de los modelos de IndexingTaskCron guardados"""
-        command = settings.READ_DATAJSON_SHELL_CMD or 'true'
         cron = cls.cron_client
 
         job_id = strings.CRONTAB_COMMENT
         for job in cron.find_comment(job_id):
             job.delete()
 
-        tasks = IndexingTaskCron.objects.filter(enabled=True)
+        tasks = TaskCron.objects.filter(enabled=True)
         for task in tasks:
-            job = cron.new(command=command, comment=job_id)
+            job = cron.new(command=task.task_script_path, comment=job_id)
 
             job.minute.on(task.time.minute)
             job.hour.on(task.time.hour)

series_tiempo_ar_api/apps/management/tests/cron_tests.py

@@ -4,14 +4,15 @@
 import mock
 from django.test import TestCase
 
-from series_tiempo_ar_api.apps.management.models import IndexingTaskCron
+from series_tiempo_ar_api.apps.management.models import TaskCron
 
 
 class CronTests(TestCase):
+    script = '/bin/true'
 
     def test_cron_added(self):
         mock_write = mock.Mock(return_value=None)
-        cron = IndexingTaskCron(time='00:00:00')
+        cron = TaskCron(time='00:00:00', task_script_path=self.script)
         cron.cron_client.write = mock_write
 
         cron.save()
@@ -20,7 +21,7 @@ def test_cron_added(self):
 
     def test_cron_removed(self):
         mock_write = mock.Mock(return_value=None)
-        cron = IndexingTaskCron(time='00:00:00')
+        cron = TaskCron(time='00:00:00', task_script_path=self.script)
         cron.cron_client.write = mock_write
 
         cron.save()
@@ -34,7 +35,7 @@ def test_cron_scheduled_correctly(self):
         minute = 0
         second = 0
         mock_write = mock.Mock(return_value=None)
-        cron = IndexingTaskCron(time=time(hour=hour, minute=minute, second=second))
+        cron = TaskCron(time=time(hour=hour, minute=minute, second=second), task_script_path=self.script)
         cron.cron_client.write = mock_write
 
         cron.save()
@@ -47,7 +48,7 @@ def test_cron_scheduled_correctly(self):
     def test_cron_weekdays(self):
 
         mock_write = mock.Mock(return_value=None)
-        cron = IndexingTaskCron(time='00:00:00', weekdays_only=True)
+        cron = TaskCron(time='00:00:00', weekdays_only=True, task_script_path=self.script)
         cron.cron_client.write = mock_write
 
         cron.save()

series_tiempo_ar_api/apps/metadata/admin.py

@@ -6,7 +6,8 @@
 from django_datajsonar.admin import FieldAdmin, DistributionAdmin
 from django_datajsonar.models import Field, Distribution
 
-from .models import IndexMetadataTask, CatalogAlias, Synonym
+from series_tiempo_ar_api.libs.singleton_admin import SingletonAdmin
+from .models import IndexMetadataTask, CatalogAlias, Synonym, MetadataConfig
 from .indexer.metadata_indexer import run_metadata_indexer
 from .utils import delete_metadata
 
@@ -81,3 +82,8 @@ def delete_model(self, _, queryset):
         fields = Field.objects.filter(distribution__identifier__in=queryset.values_list('identifier', flat=True))
         delete_metadata(list(fields))
         queryset.delete()
+
+
+@admin.register(MetadataConfig)
+class MetadataConfigAdmin(SingletonAdmin):
+    pass

series_tiempo_ar_api/apps/metadata/migrations/0003_metadataconfig.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-11-29 17:10
+from __future__ import unicode_literals
+
+import datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('metadata', '0002_auto_20180813_1510'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MetadataConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('time', models.TimeField(default=datetime.time(0, 0), help_text='Los segundos serán ignorados')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

series_tiempo_ar_api/apps/metadata/models.py

@@ -1,8 +1,14 @@
 #! coding: utf-8
+import datetime
 from typing import Sequence, List
+
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
 from django_datajsonar.models import AbstractTask, Node
+from solo.models import SingletonModel
+
+from series_tiempo_ar_api.apps.management.models import TaskCron
 
 
 class IndexMetadataTask(AbstractTask):
@@ -31,3 +37,14 @@ class Synonym(models.Model):
     terms = models.TextField(
         help_text='Lista de términos similares, separados por coma, sin espacios ni mayúsculas.'
         ' Ejemplo "ipc,inflacion"', unique=True)
+
+
+class MetadataConfig(SingletonModel):
+    SCRIPT_PATH = settings.INDEX_METADATA_SCRIPT_PATH
+
+    time = models.TimeField(help_text='Los segundos serán ignorados', default=datetime.time(hour=0, minute=0))
+
+    def save(self, *args, **kwargs):
+        super(MetadataConfig, self).save(*args, **kwargs)
+        TaskCron.objects.update_or_create(task_script_path=self.SCRIPT_PATH,
+                                          defaults={'time': self.time})

series_tiempo_ar_api/libs/singleton_admin.py

@@ -0,0 +1,6 @@
+from solo.admin import SingletonModelAdmin
+
+
+class SingletonAdmin(SingletonModelAdmin):
+    # django-des overridea el change_form_template de la clase padre(!), volvemos al default de django
+    change_form_template = 'admin/change_form.html'