From b529912796f9f7a329c28a10c78132c93eed3a24 Mon Sep 17 00:00:00 2001 From: Lucas Lavandeira <19612265+lucaslavandeira@users.noreply.github.com> Date: Thu, 29 Nov 2018 12:04:26 -0300 Subject: [PATCH] =?UTF-8?q?Borrado=20de=20c=C3=B3digo=20no=20usado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Saca toda la funcionalidad de armado de dump de analytics, la funcionalidad está provista ahora de manera más robusta por api-gateway. Saca el endpoint de save() de analytics, el cargado también es hecho por api-gateway --- .../management/commands/export_analytics.py | 10 --- series_tiempo_ar_api/apps/analytics/tasks.py | 37 --------- .../admin/analytics/change_list.html | 7 -- .../apps/analytics/templates/export.html | 21 ----- .../apps/analytics/tests/export_tests.py | 47 ----------- .../apps/analytics/tests/tests.py | 81 ------------------- .../apps/analytics/tests/view_tests.py | 48 ----------- series_tiempo_ar_api/apps/analytics/urls.py | 4 - series_tiempo_ar_api/apps/analytics/views.py | 50 ------------ 9 files changed, 305 deletions(-) delete mode 100644 series_tiempo_ar_api/apps/analytics/management/commands/export_analytics.py delete mode 100644 series_tiempo_ar_api/apps/analytics/templates/admin/analytics/change_list.html delete mode 100644 series_tiempo_ar_api/apps/analytics/templates/export.html delete mode 100644 series_tiempo_ar_api/apps/analytics/tests/export_tests.py delete mode 100644 series_tiempo_ar_api/apps/analytics/tests/tests.py delete mode 100644 series_tiempo_ar_api/apps/analytics/tests/view_tests.py diff --git a/series_tiempo_ar_api/apps/analytics/management/commands/export_analytics.py b/series_tiempo_ar_api/apps/analytics/management/commands/export_analytics.py deleted file mode 100644 index 0e0c01d8..00000000 --- a/series_tiempo_ar_api/apps/analytics/management/commands/export_analytics.py +++ /dev/null @@ -1,10 +0,0 @@ -#! coding: utf-8 - -from django.core.management import BaseCommand -from series_tiempo_ar_api.apps.analytics.tasks import export - - -class Command(BaseCommand): - - def handle(self, *args, **options): - export.delay() diff --git a/series_tiempo_ar_api/apps/analytics/tasks.py b/series_tiempo_ar_api/apps/analytics/tasks.py index fb4bd558..eef74978 100644 --- a/series_tiempo_ar_api/apps/analytics/tasks.py +++ b/series_tiempo_ar_api/apps/analytics/tasks.py @@ -1,46 +1,9 @@ #! coding: utf-8 -import os -import json import requests -import unicodecsv from django_rq import job -from django.conf import settings -from django.utils import timezone - -from .models import Query -from .utils import kong_milliseconds_to_tzdatetime from .importer import AnalyticsImporter -@job("default") -def analytics(ids, args_string, ip_address, params, timestamp_milliseconds): - params_json = json.dumps(params) - timestamp = kong_milliseconds_to_tzdatetime(timestamp_milliseconds) - query = Query(ids=ids, args=args_string, ip_address=ip_address, params=params_json, timestamp=timestamp) - query.save() - - -@job("default", timeout=360) -def export(path=None): - queryset = Query.objects.all() - filepath = path or os.path.join(settings.PROTECTED_MEDIA_DIR, settings.ANALYTICS_CSV_FILENAME) - - fields = { - 'timestamp': lambda x: timezone.localtime(x.timestamp), - 'ip_address': lambda x: x.ip_address, - 'ids': lambda x: x.ids, - 'params': lambda x: x.params, - } - - with open(filepath, 'wb') as f: - writer = unicodecsv.writer(f) - # header - writer.writerow([field for field in fields]) - for query in queryset.iterator(): - - writer.writerow([val(query) for val in fields.values()]) - - @job('default', timeout=1000) def import_analytics_from_api_mgmt(limit=1000, requests_lib=requests, import_all=False): AnalyticsImporter(limit, requests_lib).run(import_all) diff --git a/series_tiempo_ar_api/apps/analytics/templates/admin/analytics/change_list.html b/series_tiempo_ar_api/apps/analytics/templates/admin/analytics/change_list.html deleted file mode 100644 index 0e1762f5..00000000 --- a/series_tiempo_ar_api/apps/analytics/templates/admin/analytics/change_list.html +++ /dev/null @@ -1,7 +0,0 @@ -{% extends "admin/change_list.html" %} - -{% block object-tools-items %} -
OK! En unos segundos el link de descarga ({% url 'analytics:read_analytics' %}) estará actualizado.
-Redirecting in 5 seconds...
- - - \ No newline at end of file diff --git a/series_tiempo_ar_api/apps/analytics/tests/export_tests.py b/series_tiempo_ar_api/apps/analytics/tests/export_tests.py deleted file mode 100644 index b32d2e7f..00000000 --- a/series_tiempo_ar_api/apps/analytics/tests/export_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -#!coding=utf8 -import os - -import csv -from django.utils.timezone import localtime -from iso8601 import iso8601 -from django.test import TestCase -from django.utils import timezone - -from series_tiempo_ar_api.apps.analytics.models import Query -from series_tiempo_ar_api.apps.analytics.tasks import export - - -class ExportTests(TestCase): - - filepath = 'test' - - def test_export(self): - Query(args='test', params='test', ip_address='ip_addr', timestamp=timezone.now(), ids='').save() - - export(path=self.filepath) - - with open(self.filepath) as f: - self.assertEqual(len(f.readlines()), 2) - - def test_export_empty(self): - export(path=self.filepath) - - with open(self.filepath) as f: - # Esperado solo una línea de header - self.assertEqual(len(f.readlines()), 1) - - def tearDown(self): - if os.path.exists(self.filepath): - os.remove(self.filepath) - - def test_export_dates(self): - query = Query(args='test', params='test', ip_address='ip_addr', timestamp=timezone.now(), ids='') - query.save() - - export(path=self.filepath) - with open(self.filepath) as f: - next(f) - for line in csv.reader(f): - date = iso8601.parse_date(line[0]) - # Timestamp del modelo en UTC, pasandola a localtime debe ser igual a la del CSV - self.assertEqual(localtime(query.timestamp), date) diff --git a/series_tiempo_ar_api/apps/analytics/tests/tests.py b/series_tiempo_ar_api/apps/analytics/tests/tests.py deleted file mode 100644 index ee4c0209..00000000 --- a/series_tiempo_ar_api/apps/analytics/tests/tests.py +++ /dev/null @@ -1,81 +0,0 @@ -#! coding: utf-8 -import json -import copy - -from django.test import TestCase -from django.urls import reverse - -from series_tiempo_ar_api.apps.analytics.tasks import analytics -from series_tiempo_ar_api.apps.analytics.models import Query -from series_tiempo_ar_api.apps.analytics.utils import kong_milliseconds_to_tzdatetime - - -class AnalyticsTests(TestCase): - - def test_analytics_creates_model(self): - series_ids = "serie1" - args = "query_arg=value" - ip_address = "ip_address" - args_dict = {'json': 'with_args'} - timestamp = 1433209822425 - - # Expected: la función crea un objeto Query en la base de datos - analytics(series_ids, args, ip_address, args_dict, timestamp) - - query = Query.objects.get(ids=series_ids, - args=args, - ip_address=ip_address) - - timestamp_datetime = kong_milliseconds_to_tzdatetime(timestamp) - - self.assertTrue(query) - self.assertEqual(args_dict, json.loads(query.params)) - self.assertEqual(timestamp_datetime, query.timestamp) - - -class AnalyticsViewTests(TestCase): - - body = { - 'request': { - 'querystring': {'ids': 'test_id'}, - 'uri': 'api/series', - 'request_uri': 'http://localhost:8000/api/series?ids=test_id', - }, - 'client_ip': '127.0.0.1', - 'started_at': 1271237123, - } - - def test_view_valid_body(self): - count_before = Query.objects.count() - response = self.client.post(reverse('analytics:save'), - json.dumps(self.body), - content_type="application/json") - - self.assertEqual(response.status_code, 200) - self.assertEqual(Query.objects.count(), count_before + 1) - - def test_view_invalid_method(self): - response = self.client.put(reverse('analytics:save'), - json.dumps(self.body), - content_type='application/json') - - self.assertEqual(response.status_code, 405) - - def test_view_empty_body(self): - response = self.client.post(reverse('analytics:save'), - '{}', - content_type='application/json') - - self.assertEqual(response.status_code, 400) - - def test_admin_call_not_logged(self): - body = copy.deepcopy(self.body) - body['request']['uri'] = 'admin/api/dataset/12' - count_before = Query.objects.count() - response = self.client.post(reverse('analytics:save'), - json.dumps(body), - content_type="application/json") - self.assertEqual(response.status_code, 200) - - # Count unchanged - self.assertEqual(Query.objects.count(), count_before) diff --git a/series_tiempo_ar_api/apps/analytics/tests/view_tests.py b/series_tiempo_ar_api/apps/analytics/tests/view_tests.py deleted file mode 100644 index 2262c0f9..00000000 --- a/series_tiempo_ar_api/apps/analytics/tests/view_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -#!coding=utf8 -import os -import mock - -import sendfile -from django.http import FileResponse -from django.test import TestCase -from django.urls import reverse -from django.contrib.auth.models import User - -samples_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'samples') - - -class ExportViewTests(TestCase): - - def test_export_when_not_staff(self): - response = self.client.get(reverse('analytics:export_analytics')) - - # Expected: admin login redirect - self.assertEqual(response.status_code, 302) - - def test_export_as_staff(self): - user = User(username='user', password='pass', email='mail@test.com', is_staff=True) - user.save() - self.client.force_login(user) - response = self.client.get(reverse('analytics:export_analytics')) - - self.assertEqual(response.status_code, 200) - - -class AnalyticsDownloadTests(TestCase): - - def test_download_when_not_staff(self): - response = self.client.get(reverse('analytics:read_analytics')) - - # Expected: admin login redirect - self.assertEqual(response.status_code, 302) - - def test_download_as_staff(self): - user = User(username='user', password='pass', email='mail@test.com', is_staff=True) - user.save() - self.client.force_login(user) - - response_file = open(os.path.join(samples_dir, 'sample_analytics_dump.csv')) - sendfile.sendfile = mock.Mock(return_value=FileResponse(response_file)) - response = self.client.get(reverse('analytics:read_analytics')) - - self.assertEqual(response.status_code, 200) diff --git a/series_tiempo_ar_api/apps/analytics/urls.py b/series_tiempo_ar_api/apps/analytics/urls.py index b08434e9..ceb170ff 100644 --- a/series_tiempo_ar_api/apps/analytics/urls.py +++ b/series_tiempo_ar_api/apps/analytics/urls.py @@ -1,8 +1,4 @@ from django.conf.urls import url -from .views import save, read_analytics, export_analytics urlpatterns = [ - url('^save/$', save, name='save'), - url('^analytics.csv', read_analytics, name='read_analytics'), - url('^export', export_analytics, name='export_analytics'), ] diff --git a/series_tiempo_ar_api/apps/analytics/views.py b/series_tiempo_ar_api/apps/analytics/views.py index 2e329dd0..a2b5079b 100644 --- a/series_tiempo_ar_api/apps/analytics/views.py +++ b/series_tiempo_ar_api/apps/analytics/views.py @@ -1,52 +1,2 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - -import os -import json - -import sendfile -from django.http import HttpResponse -from django.conf import settings -from django.views.decorators.csrf import csrf_exempt -from django.contrib.admin.views.decorators import staff_member_required -from django.shortcuts import render - -from .tasks import analytics, export - - -@csrf_exempt -def save(request): - if request.method != 'POST': - return HttpResponse(status=405) - - body = json.loads(request.body) - req_data = body.get('request') - if not req_data: # Fatal error - return HttpResponse(status=400) - - uri = req_data.get('uri') - if 'admin/' in uri or 'api/' not in uri: - return HttpResponse() - - params = req_data.get('querystring') - ids = params.get('ids', 'No especificado') - ip_address = body.get('client_ip') - timestamp = body.get('started_at') - args = req_data.get('request_uri', 'No especificado') - - params_start = args.find('?') - if params_start > 0: - args = args[params_start:] - analytics.delay(ids, args, ip_address, params, timestamp) - return HttpResponse() - - -@staff_member_required -def read_analytics(request): - return sendfile.sendfile(request, os.path.join(settings.PROTECTED_MEDIA_DIR, 'analytics.csv')) - - -@staff_member_required -def export_analytics(request): - export.delay() - return HttpResponse(render(request, 'export.html'))