From fe93f37c038b168e8c808409cab635c608c2e084 Mon Sep 17 00:00:00 2001 From: Tony Thomas <01tonythomas@gmail.com> Date: Sun, 14 Oct 2018 12:51:11 +0200 Subject: [PATCH 1/3] Use proper query params to build filter URLS for bot reports --- dashboard/core/bots/reports/views.py | 164 +++++++----------- .../dashboard/dashboard.module.js | 43 ++--- .../dashboard/dashboard.services.js | 4 +- dashboard/urls.py | 8 +- 4 files changed, 80 insertions(+), 139 deletions(-) diff --git a/dashboard/core/bots/reports/views.py b/dashboard/core/bots/reports/views.py index bb5748b..e273977 100644 --- a/dashboard/core/bots/reports/views.py +++ b/dashboard/core/bots/reports/views.py @@ -1,4 +1,9 @@ -import json, urllib.request, urllib.parse, urllib.error, logging, sys +import json +import urllib.request +import urllib.parse +import urllib.error +import logging +import sys from datetime import datetime, timedelta from django.http import HttpResponse, HttpResponseBadRequest @@ -34,119 +39,70 @@ db_character_separator = '\\' -class BotDataReportImprovementListView(ListAPIView): - """Fetch improvements for main page tables""" - - model = BotReportData +class ReportDataBaseListViewSerializer(ListAPIView): serializer_class = BotReportDataSerializer - queryset = BotReportData.objects.filter(aggregation='None') - def get_queryset(self): - try: - days_since = int(self.kwargs.get('days_since')) - except ValueError: - days_since = int(5) + def build_filters(self, is_improvement=True) -> dict: + days_since = int(self.request.query_params.get('days_since', 5)) - if self.kwargs.get('platform') == 'all': - platform_obj = Platform.objects.all() - else: - platform_obj = Platform.objects.filter(pk=self.kwargs.get('platform')) + platform_filter = self.request.query_params.get('platform', None) + platform_filter_by = {'pk': platform_filter} if platform_filter else {} + platforms = Platform.objects.filter(**platform_filter_by) - if self.kwargs.get('cpu') == 'all': - cpu_obj = CPUArchitecture.objects.all() - else: - cpu_obj = CPUArchitecture.objects.filter(pk=self.kwargs.get('cpu')) + cpu_filter = self.request.query_params.get('cpu', None) + cpu_filter_by = {'pk': cpu_filter} if cpu_filter else {} + cpus = CPUArchitecture.objects.filter(**cpu_filter_by) - if self.kwargs.get('gpu') == 'all': - gpu_obj = GPUType.objects.all() - else: - gpu_obj = GPUType.objects.filter(pk=self.kwargs.get('gpu')) + gpu_filter = self.request.query_params.get('gpu', None) + gpu_filter_by = {'pk': gpu_filter} if gpu_filter else {} + gpus = GPUType.objects.filter(**gpu_filter_by) - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter(pk=self.kwargs.get('browser')) + browser_filter = self.request.query_params.get('browser', None) + browser_filter_by = {'pk': browser_filter} if browser_filter else {} + browsers = Browser.objects.filter(**browser_filter_by) - if self.kwargs.get('test') == 'all': - root_test = Test.objects.all() - else: - root_test = Test.objects.filter(pk=self.kwargs.get('test')) + test_filter = self.request.query_params.get('test', None) + test_filter_by = {'pk': test_filter} if test_filter else {} + tests = Test.objects.filter(**test_filter_by) - if self.kwargs.get('bot') == 'all': - bot_obj = Bot.objects.all() - else: - bot_obj = Bot.objects.filter(pk=self.kwargs.get('bot')) + bot_filter = self.request.query_params.get('bot', None) + bot_filter_by = {'pk': bot_filter} if bot_filter else {} + bots = Bot.objects.filter(**bot_filter_by) - limit = int(self.kwargs.get('limit')) - bot = bot_obj.filter( - platform__in=platform_obj, cpuArchitecture__in=cpu_obj, - gpuType__in=gpu_obj, enabled=True + bot = bots.filter( + platform__in=platforms, cpuArchitecture__in=cpus, + gpuType__in=gpus, enabled=True ) - requested_time = datetime.utcnow() + timedelta(days=-days_since) - queryset = super(BotDataReportImprovementListView, self).get_queryset() - return queryset.filter( - aggregation='None', timestamp__gt=requested_time, - root_test__in=root_test, bot__in=bot, browser__in=browser_obj, - is_improvement=True - ).order_by('-delta')[:limit] - + return { + 'timestamp__gt': requested_time, + 'root_test__in': tests, + 'bot__in': bot, + 'browser__in': browsers, + 'is_improvement': is_improvement + } -class BotDataReportRegressionListView(ListAPIView): - """Fetch regressions for main page tables""" +class BotDataReportImprovementListView(ReportDataBaseListViewSerializer): model = BotReportData - serializer_class = BotReportDataSerializer queryset = BotReportData.objects.filter(aggregation='None') def get_queryset(self): - try: - days_since = int(self.kwargs.get('days_since')) - except ValueError: - days_since = int(5) - - if self.kwargs.get('platform') == 'all': - platform_obj = Platform.objects.all() - else: - platform_obj = Platform.objects.filter(pk=self.kwargs.get('platform')) - - if self.kwargs.get('cpu') == 'all': - cpu_obj = CPUArchitecture.objects.all() - else: - cpu_obj = CPUArchitecture.objects.filter(pk=self.kwargs.get('cpu')) - - if self.kwargs.get('gpu') == 'all': - gpu_obj = GPUType.objects.all() - else: - gpu_obj = GPUType.objects.filter(pk=self.kwargs.get('gpu')) - - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter(pk=self.kwargs.get('browser')) + queryset = super(BotDataReportImprovementListView, self).get_queryset() + filter_params = self.build_filters(is_improvement=True) + limit = int(self.kwargs.get('limit', 10)) + return queryset.filter(**filter_params).order_by('-delta')[:limit] - if self.kwargs.get('test') == 'all': - root_test = Test.objects.all() - else: - root_test = Test.objects.filter(pk=self.kwargs.get('test')) - if self.kwargs.get('bot') == 'all': - bot_obj = Bot.objects.all() - else: - bot_obj = Bot.objects.filter(pk=self.kwargs.get('bot')) +class BotDataReportRegressionListView(ReportDataBaseListViewSerializer): + model = BotReportData + queryset = BotReportData.objects.filter(aggregation='None') - limit = int(self.kwargs.get('limit')) - bot = bot_obj.filter( - platform__in=platform_obj, cpuArchitecture__in=cpu_obj, - gpuType__in=gpu_obj, enabled=True - ) - requested_time = datetime.utcnow() + timedelta(days=-days_since) + def get_queryset(self): queryset = super(BotDataReportRegressionListView, self).get_queryset() - return queryset.filter( - aggregation='None', timestamp__gt=requested_time, - root_test__in=root_test, bot__in=bot, browser__in=browser_obj, - is_improvement=False - ).order_by('-delta')[:limit] + filter_params = self.build_filters(is_improvement=False) + limit = int(self.kwargs.get('limit', 10)) + return queryset.filter(**filter_params).order_by('-delta')[:limit] class TestPathList(ListAPIView): @@ -228,7 +184,8 @@ def calculate_prefix(cls, munits, mean_value, curr_string, original_prefix): if len(munits) == 1: if mean_value > prefix['unit']: mean_value = mean_value / prefix['unit'] - curr_string += format(mean_value, '.2f') + " " + original_prefix + curr_string += format(mean_value, '.2f') + \ + " " + original_prefix return curr_string munits = munits[index + 1:] @@ -289,7 +246,7 @@ def post(self, request, format=None): bot_id = self.request.POST.get('bot_id') json_data = self.request.POST.get('test_data') except AttributeError: - log.error("Got invalid params from the bot: %s"% request.auth) + log.error("Got invalid params from the bot: %s" % request.auth) return HttpResponseBadRequest( "Some params are missing in the request" ) @@ -305,14 +262,14 @@ def post(self, request, format=None): test_data = json.loads(json_data) except AttributeError: return HttpResponseBadRequest( - "Error parsing JSON file from bot: %s "% request.auth + "Error parsing JSON file from bot: %s " % request.auth ) bot = Bot.objects.get(pk=bot_id) if not bot.enabled: log.error("Got data from disabled bot: %s" % bot_id) - return HttpResponseBadRequest("The bot %s is not enabled"% bot_id) + return HttpResponseBadRequest("The bot %s is not enabled" % bot_id) try: browser = Browser.objects.get(pk=browser_id) @@ -327,11 +284,11 @@ def post(self, request, format=None): except Test.DoesNotExist: log.error( "Got invalid root test: %s from bot: %s for browser: %s, " - "browser_version: %s" %( + "browser_version: %s" % ( test_id, bot_id, browser_id, browser_version )) return HttpResponseBadRequest( - "The test %s does not exist"% test_id + "The test %s does not exist" % test_id ) test_data_results = BenchmarkResults(test_data) @@ -355,7 +312,8 @@ def post(self, request, format=None): original_prefix=current_metric.unit ) else: - modified_prefix = str(mean_value) + " " + current_metric.unit + modified_prefix = str(mean_value) + \ + " " + current_metric.unit except MetricUnit.DoesNotExist: log.error( "Got wrong Metric %s for bot: %s, browser: %s, " @@ -365,7 +323,7 @@ def post(self, request, format=None): ) ) return HttpResponseBadRequest( - "The Metric Unit %s does not exist"% metric_name + "The Metric Unit %s does not exist" % metric_name ) if current_metric.unit != unit: @@ -373,11 +331,11 @@ def post(self, request, format=None): "bot: %s, browser: %s, browser_version: %s, " "root_test: %s, test_description: %s" % (unit, metric_name, bot_id, browser_id, - browser_version,test_id, raw_path) + browser_version, test_id, raw_path) ) return HttpResponseBadRequest("The received unit: %s field of " "Metric Unit %s does not match" - % (unit,metric_name) + % (unit, metric_name) ) if self.is_aggregated(metric=result['metric']): diff --git a/dashboard/static/js/app_controllers/dashboard/dashboard.module.js b/dashboard/static/js/app_controllers/dashboard/dashboard.module.js index 82a2b8e..2144f05 100644 --- a/dashboard/static/js/app_controllers/dashboard/dashboard.module.js +++ b/dashboard/static/js/app_controllers/dashboard/dashboard.module.js @@ -72,39 +72,26 @@ app.controller( $scope.noImprovementsFound = false; $scope.noRegressionsFound = false; $scope.loading = true; - $scope.selectedDays = !$scope.selectedDays ? 5 : $scope.selectedDays; - $scope.listLimit = !$scope.listLimit? 10 : $scope.listLimit; - $scope.selectedBrowserId = !$scope.selectedBrowser ? 'all' : $scope.selectedBrowser.id; - $scope.selectedPlatformId = !$scope.selectedPlatform ? 'all' : $scope.selectedPlatform.id; - $scope.selectedCPUId = !$scope.selectedCPU ? 'all' : $scope.selectedCPU.id; - $scope.selectedGPUId = !$scope.selectedGPU ? 'all' : $scope.selectedGPU.id; - $scope.selectedTestId = !$scope.selectedTest ? 'all' : $scope.selectedTest.root_test.id; - $scope.selectedBotName = !$scope.selectedBot ? 'all' : $scope.selectedBot.name; - $scope.improvement_reports = botReportsImprovementFactory.query({ - days_since: $scope.selectedDays, - platform: $scope.selectedPlatformId, - gpu: $scope.selectedGPUId, - cpu: $scope.selectedCPUId, - browser: $scope.selectedBrowserId, - test: $scope.selectedTestId, - bot: $scope.selectedBotName, - limit: $scope.listLimit - }, function (data) { + $scope.query_params = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + platform: !$scope.selectedPlatform ? undefined : $scope.selectedPlatform.id, + gpu: !$scope.selectedGPU ? undefined : $scope.selectedGPU.id, + cpu: !$scope.selectedCPU ? undefined : $scope.selectedCPU.id, + test: !$scope.selectedTest ? undefined : $scope.selectedTest.root_test.id, + bot: !$scope.selectedBot ? undefined : $scope.selectedBot.name, + days_since: !$scope.selectedDays ? 5 : $scope.selectedDays, + limit: !$scope.listLimit? 10 : $scope.listLimit + }); + + $scope.improvement_reports = botReportsImprovementFactory.query( + $scope.query_params, function (data) { if (data.length == 0) { $scope.noImprovementsFound = true; } $scope.loading_improvements = false; }); - $scope.regression_reports = botReportsRegressionFactory.query({ - days_since: $scope.selectedDays, - platform: $scope.selectedPlatformId, - gpu: $scope.selectedGPUId, - cpu: $scope.selectedCPUId, - browser: $scope.selectedBrowserId, - test: $scope.selectedTestId, - bot: $scope.selectedBotName, - limit: $scope.listLimit - }, function (data) { + $scope.regression_reports = botReportsRegressionFactory.query( + $scope.query_params, function (data) { if (data.length == 0) { $scope.noRegressionsFound = true; } diff --git a/dashboard/static/js/app_controllers/dashboard/dashboard.services.js b/dashboard/static/js/app_controllers/dashboard/dashboard.services.js index 7b2c8e8..c15bd4a 100644 --- a/dashboard/static/js/app_controllers/dashboard/dashboard.services.js +++ b/dashboard/static/js/app_controllers/dashboard/dashboard.services.js @@ -6,11 +6,11 @@ app = angular.module('browserperfdash.dashboard.services', ['ngResource']); app.factory('botReportsImprovementFactory', function($resource) { - return $resource('/dash/report/improvement/:days_since/:platform/:gpu/:cpu/:browser/:test/:bot/:limit'); + return $resource('/dash/report/improvements'); }); app.factory('botReportsRegressionFactory', function($resource) { - return $resource('/dash/report/regression/:days_since/:platform/:gpu/:cpu/:browser/:test/:bot/:limit'); + return $resource('/dash/report/regressions'); }); app.factory('browserFactory', function($resource) { diff --git a/dashboard/urls.py b/dashboard/urls.py index 842d2e5..d710d9e 100644 --- a/dashboard/urls.py +++ b/dashboard/urls.py @@ -30,10 +30,6 @@ url(r'^tests_for_browser_bot/(?P.+)/(?P.*)$', TestsForBrowserBotList.as_view()), url(r'^results_for_subtest/(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P.+)/$', ResultsForSubtestList.as_view()), - url(r'^report/improvement/(?P\d+)/(?P[-\w]+)/(?P[-\w]+)/' - r'(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P\d+)/$', - BotDataReportImprovementListView.as_view()), - url(r'^report/regression/(?P\d+)/(?P[-\w]+)/(?P[-\w]+)/' - r'(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P\d+)/$', - BotDataReportRegressionListView.as_view()) + url(r'^report/improvements', BotDataReportImprovementListView.as_view()), + url(r'^report/regressions', BotDataReportRegressionListView.as_view()) ] From b9db17af2823a4c4fb56fc37deb789d0dd845a8d Mon Sep 17 00:00:00 2001 From: Tony Thomas <01tonythomas@gmail.com> Date: Sun, 14 Oct 2018 11:28:12 +0200 Subject: [PATCH 2/3] fix(#26) No more results with no prev results --- dashboard/core/bots/reports/views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dashboard/core/bots/reports/views.py b/dashboard/core/bots/reports/views.py index e273977..ef21e3e 100644 --- a/dashboard/core/bots/reports/views.py +++ b/dashboard/core/bots/reports/views.py @@ -79,7 +79,7 @@ def build_filters(self, is_improvement=True) -> dict: 'root_test__in': tests, 'bot__in': bot, 'browser__in': browsers, - 'is_improvement': is_improvement + 'is_improvement': is_improvement, } @@ -91,7 +91,8 @@ def get_queryset(self): queryset = super(BotDataReportImprovementListView, self).get_queryset() filter_params = self.build_filters(is_improvement=True) limit = int(self.kwargs.get('limit', 10)) - return queryset.filter(**filter_params).order_by('-delta')[:limit] + return queryset.filter(**filter_params).exclude( + prev_result__isnull=True).order_by('-delta')[:limit] class BotDataReportRegressionListView(ReportDataBaseListViewSerializer): @@ -102,7 +103,8 @@ def get_queryset(self): queryset = super(BotDataReportRegressionListView, self).get_queryset() filter_params = self.build_filters(is_improvement=False) limit = int(self.kwargs.get('limit', 10)) - return queryset.filter(**filter_params).order_by('-delta')[:limit] + return queryset.filter(**filter_params).exclude( + prev_result__isnull=True).order_by('-delta')[:limit] class TestPathList(ListAPIView): From c2bc39a0044b02cff436dc67d5ccb86b318fb883 Mon Sep 17 00:00:00 2001 From: Tony Thomas <01tonythomas@gmail.com> Date: Sun, 14 Oct 2018 12:52:25 +0200 Subject: [PATCH 3/3] Use proper filter params for all URLS --- dashboard/core/bots/reports/serializers.py | 2 +- dashboard/core/bots/reports/views.py | 164 +++++++++--------- dashboard/core/bots/serializers.py | 1 - dashboard/core/bots/views.py | 37 +--- .../dashboard/dashboard.module.js | 27 +-- .../dashboard/dashboard.services.js | 12 +- .../js/app_controllers/plot/plot.module.js | 65 +++---- .../js/app_controllers/plot/plot.services.js | 12 +- dashboard/urls.py | 26 +-- docs/local-settings.py | 2 - 10 files changed, 169 insertions(+), 179 deletions(-) diff --git a/dashboard/core/bots/reports/serializers.py b/dashboard/core/bots/reports/serializers.py index 8e25fc1..145b20e 100644 --- a/dashboard/core/bots/reports/serializers.py +++ b/dashboard/core/bots/reports/serializers.py @@ -8,7 +8,7 @@ from dashboard.core.tests.serializers import TestListListSerializer -class TestPathListSerializer(serializers.Serializer): +class TestPathsListSerializer(serializers.Serializer): test_path = serializers.CharField() root_test = serializers.CharField() aggregation = serializers.CharField() diff --git a/dashboard/core/bots/reports/views.py b/dashboard/core/bots/reports/views.py index ef21e3e..3b2dc90 100644 --- a/dashboard/core/bots/reports/views.py +++ b/dashboard/core/bots/reports/views.py @@ -24,7 +24,7 @@ from dashboard.core.cpus.models import CPUArchitecture from dashboard.core.gpus.models import GPUType -from dashboard.core.bots.reports.serializers import TestPathListSerializer, \ +from dashboard.core.bots.reports.serializers import TestPathsListSerializer, \ TestsForBrowserBotListSerializer, ResultsForSubtestListSerializer, \ BotReportDataSerializer @@ -39,7 +39,7 @@ db_character_separator = '\\' -class ReportDataBaseListViewSerializer(ListAPIView): +class BaseReportDataListViewSerializer(ListAPIView): serializer_class = BotReportDataSerializer def build_filters(self, is_improvement=True) -> dict: @@ -83,7 +83,7 @@ def build_filters(self, is_improvement=True) -> dict: } -class BotDataReportImprovementListView(ReportDataBaseListViewSerializer): +class BotDataReportImprovementListView(BaseReportDataListViewSerializer): model = BotReportData queryset = BotReportData.objects.filter(aggregation='None') @@ -95,7 +95,7 @@ def get_queryset(self): prev_result__isnull=True).order_by('-delta')[:limit] -class BotDataReportRegressionListView(ReportDataBaseListViewSerializer): +class BotDataReportRegressionListView(BaseReportDataListViewSerializer): model = BotReportData queryset = BotReportData.objects.filter(aggregation='None') @@ -107,18 +107,21 @@ def get_queryset(self): prev_result__isnull=True).order_by('-delta')[:limit] -class TestPathList(ListAPIView): - serializer_class = TestPathListSerializer +class TestPathsList(ListAPIView): + serializer_class = TestPathsListSerializer def get_queryset(self): - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter(pk=self.kwargs.get('browser')) + browser_filter = self.request.query_params.get('browser', None) + browser_filter_by = {'pk': browser_filter} if browser_filter else {} + browsers = Browser.objects.filter(**browser_filter_by) + + root_test_filter = self.request.query_params.get('root_test', None) + root_test_filter_by = {'pk': root_test_filter} if root_test_filter \ + else {} + root_test = Test.objects.filter(**root_test_filter_by) return BotReportData.objects.filter( - browser__in=browser_obj, - root_test=Test.objects.filter(pk=self.kwargs.get('test')) + browser__in=browsers, root_test__in=root_test ).distinct('test_path') @@ -126,41 +129,40 @@ class TestsForBrowserBotList(ListAPIView): serializer_class = TestsForBrowserBotListSerializer def get_queryset(self): - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter(pk=self.kwargs.get('browser')) - try: - bot = Bot.objects.get(pk=self.kwargs.get('bot')) - return BotReportData.objects.filter( - browser__in=browser_obj, bot=bot - ).distinct('root_test') - except Bot.DoesNotExist: - return BotReportData.objects.filter( - browser__in=browser_obj - ).distinct('root_test') + browser_filter = self.request.query_params.get('browser', None) + browser_filter_by = {'pk': browser_filter} if browser_filter else {} + browsers = Browser.objects.filter(**browser_filter_by) + bot_filter = self.request.query_params.get('bot', None) + bot_filter_by = {'pk': bot_filter} if bot_filter else {} + bots = Bot.objects.filter(**bot_filter_by) + + return BotReportData.objects.filter( + browser__in=browsers, bot__in=bots + ).distinct('root_test') -class ResultsForSubtestList(ListAPIView): + +class ResultsForSubTestList(ListAPIView): serializer_class = ResultsForSubtestListSerializer def get_queryset(self): - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter(pk=self.kwargs.get('browser')) - test = Test.objects.get(pk=self.kwargs.get('test')) - test_path = urllib.parse.unquote(self.kwargs.get('subtest')) - if self.kwargs.get('bot') == 'all': - return BotReportData.objects.filter( - browser__in=browser_obj, root_test=test, test_path=test_path - ).order_by('timestamp') - else: - bot = Bot.objects.get(pk=self.kwargs.get('bot')) - return BotReportData.objects.filter( - browser__in=browser_obj, root_test=test, test_path=test_path, - bot=bot - ).order_by('timestamp') + browser_filter = self.request.query_params.get('browser', None) + browser_filter_by = {'pk': browser_filter} if browser_filter else {} + browsers = Browser.objects.filter(**browser_filter_by) + + test = Test.objects.get(pk=self.request.query_params.get('test')) + test_path = urllib.parse.unquote( + self.request.query_params.get('subtest') + ) + + bot_filter = self.request.query_params.get('bot', None) + bot_filter_by = {'pk': bot_filter} if bot_filter else {} + bots = Bot.objects.filter(**bot_filter_by) + + return BotReportData.objects.filter( + browser__in=browsers, root_test=test, test_path=test_path, + bot__in=bots + ).order_by('timestamp') class BotReportView(APIView): @@ -248,12 +250,13 @@ def post(self, request, format=None): bot_id = self.request.POST.get('bot_id') json_data = self.request.POST.get('test_data') except AttributeError: - log.error("Got invalid params from the bot: %s" % request.auth) + log.error( + 'Got invalid params from the bot: {0}'.format(request.auth)) return HttpResponseBadRequest( - "Some params are missing in the request" + 'Some params are missing in the request' ) - log.info("Started processing data from bot %s" % (bot_id)) + log.info('Started processing data from bot {0}'.format(bot_id)) # The timestamp may/may not be there - hence not checking timestamp = None if self.request.POST.get('timestamp'): @@ -264,33 +267,35 @@ def post(self, request, format=None): test_data = json.loads(json_data) except AttributeError: return HttpResponseBadRequest( - "Error parsing JSON file from bot: %s " % request.auth + 'Error parsing JSON file from bot: {0}'.format(request.auth) ) bot = Bot.objects.get(pk=bot_id) if not bot.enabled: - log.error("Got data from disabled bot: %s" % bot_id) - return HttpResponseBadRequest("The bot %s is not enabled" % bot_id) + log.error('Got data from disabled bot: {0}'.format(bot_id)) + return HttpResponseBadRequest( + 'The bot {0} is not enabled'.format(bot_id)) try: browser = Browser.objects.get(pk=browser_id) except Browser.DoesNotExist: log.error( - "Got invalid browser %s from bot: %s" % (browser_id, bot_id) + 'Got invalid browser {0} from bot: {1}'.format( + browser_id, bot_id) ) - return HttpResponseBadRequest("The browser does not exist") + return HttpResponseBadRequest('The browser does not exist') try: root_test = Test.objects.get(pk=test_id) except Test.DoesNotExist: log.error( - "Got invalid root test: %s from bot: %s for browser: %s, " - "browser_version: %s" % ( - test_id, bot_id, browser_id, browser_version - )) + 'Got invalid root test: {0} from bot: {1} for browser: {2}, ' + 'browser_version: {3}'.format( + test_id, bot_id, browser_id, browser_version) + ) return HttpResponseBadRequest( - "The test %s does not exist" % test_id + 'The test {0} does not exist'.format(test_id) ) test_data_results = BenchmarkResults(test_data) @@ -318,27 +323,28 @@ def post(self, request, format=None): " " + current_metric.unit except MetricUnit.DoesNotExist: log.error( - "Got wrong Metric %s for bot: %s, browser: %s, " - "browser_version: %s, root_test: %s, test_description: " - "%s" % (metric_name, bot_id, browser_id, browser_version, - test_id, raw_path - ) + 'Got wrong Metric {0} for bot: {1}, browser: {2}, ' + 'browser_version: {3}, root_test: {4}, test_description: ' + '{5}'.format( + metric_name, bot_id, browser_id, browser_version, + test_id, raw_path) ) return HttpResponseBadRequest( - "The Metric Unit %s does not exist" % metric_name + 'The Metric Unit {0} does not exist'.format(metric_name) ) if current_metric.unit != unit: - log.error("Got wrong unit %s for metric unit %s data for " - "bot: %s, browser: %s, browser_version: %s, " - "root_test: %s, test_description: %s" % - (unit, metric_name, bot_id, browser_id, - browser_version, test_id, raw_path) - ) - return HttpResponseBadRequest("The received unit: %s field of " - "Metric Unit %s does not match" - % (unit, metric_name) - ) + log.error( + 'Got wrong unit {0} for metric unit {1} data for bot: ' + '{2}, browser: {3}, browser_version: {4}, root_test: {5}, ' + 'test_description: {6}'.format( + unit, metric_name, bot_id, browser_id, + browser_version, test_id, raw_path + ) + ) + return HttpResponseBadRequest( + 'The received unit: {0} field of Metric Unit {1} does not ' + 'match'.format(unit, metric_name)) if self.is_aggregated(metric=result['metric']): aggregation = self.extract_aggregation(metric=result['metric']) @@ -365,13 +371,13 @@ def post(self, request, format=None): ) except Exception as e: log.error( - "Failed inserting data for bot: %s, browser: %s, " - "browser_version: %s, root_test: %s, test_description: %s" - " and Exception %s" % - (bot_id, browser_id, browser_version, test_id, raw_path, - str(e)) + 'Failed inserting data for bot: {0}, browser: {1}, ' + 'browser_version: {2}, root_test: {3}, test_description: ' + '{4} and Exception {5}'.format( + bot_id, browser_id, browser_version, test_id, + raw_path, e) ) - return HttpResponseBadRequest("Exception inserting the data " - "into the DB: %s" % str(e)) + return HttpResponseBadRequest( + 'Exception while inserting the data: {0}'.format(e)) - return HttpResponse("The POST went through") + return HttpResponse('The POST went through') diff --git a/dashboard/core/bots/serializers.py b/dashboard/core/bots/serializers.py index 056fb31..26f0181 100644 --- a/dashboard/core/bots/serializers.py +++ b/dashboard/core/bots/serializers.py @@ -13,7 +13,6 @@ class Meta: class BotsFullDetailsForResultsExistListSerializer(serializers.ModelSerializer): - name = serializers.CharField() platform = PlatformListSerializer() cpuArchitecture = CPUArchitectureListSerializer() gpuType = GPUTypeListSerializer() diff --git a/dashboard/core/bots/views.py b/dashboard/core/bots/views.py index c7153e8..bbdfebf 100644 --- a/dashboard/core/bots/views.py +++ b/dashboard/core/bots/views.py @@ -9,43 +9,24 @@ from rest_framework.generics import ListAPIView -class BotsForResultsExistList(ListAPIView): - """Fetch just the botname for the plot pages""" +class BaseBotResultsView(ListAPIView): model = Bot - serializer_class = BotsForResultsExistListSerializer def get_queryset(self): - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter( - pk=self.kwargs.get('browser') - ) + browser_filter = self.request.query_params.get('browser', None) + browser_filter_by = {'pk': browser_filter} if browser_filter else {} + browsers = Browser.objects.filter(**browser_filter_by) return Bot.objects.filter( name__in=BotReportData.objects.filter( - browser__in=browser_obj + browser__in=browsers ).distinct('bot').values('bot'), enabled=True ) -class BotsFullDetailsForResultsExistList(ListAPIView): - """Fetch detailed bot fields for the home page""" - model = Bot - serializer_class = BotsFullDetailsForResultsExistListSerializer - - def get_queryset(self): - if self.kwargs.get('browser') == 'all': - browser_obj = Browser.objects.all() - else: - browser_obj = Browser.objects.filter( - pk=self.kwargs.get('browser') - ) - return Bot.objects.filter( - name__in=BotReportData.objects.filter( - browser__in=browser_obj - ).distinct('bot').values('bot'), - enabled=True - ) +class BotsForResultsExistList(BaseBotResultsView): + serializer_class = BotsForResultsExistListSerializer +class BotsFullDetailsForResultsExistList(BaseBotResultsView): + serializer_class = BotsFullDetailsForResultsExistListSerializer diff --git a/dashboard/static/js/app_controllers/dashboard/dashboard.module.js b/dashboard/static/js/app_controllers/dashboard/dashboard.module.js index 2144f05..f15e983 100644 --- a/dashboard/static/js/app_controllers/dashboard/dashboard.module.js +++ b/dashboard/static/js/app_controllers/dashboard/dashboard.module.js @@ -22,16 +22,16 @@ app.controller( $sce, $filter ) { $scope.browsers = browserFactory.query(); - $scope.bots = botFullDetailsForResultsExistFactory.query({ - browser: 'all' - }); + $scope.bots = botFullDetailsForResultsExistFactory.query({}); $scope.platforms = platformFactory.query(); $scope.gpus = gpuFactory.query(); $scope.cpus = cpuArchFactory.query(); - $scope.tests = testsForBrowserAndBotFactory.query({ - browser: !$scope.selectedBrowser ? 'all' : $scope.selectedBrowser.id, - bot: !$scope.selectedBot ? null : $scope.selectedBot.name, + + $scope.tests_query = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + bot: !$scope.selectedBot ? undefined : $scope.selectedBot.name, }); + $scope.tests = testsForBrowserAndBotFactory.query($scope.tests_query); $scope.botDetailsPopover = { templateUrl: 'bot-template.html' }; @@ -46,13 +46,16 @@ app.controller( }; $scope.updateOthersOnBrowserChange = function () { //There can be chance of test change - $scope.tests = testsForBrowserAndBotFactory.query({ - browser: !$scope.selectedBrowser ? 'all' : $scope.selectedBrowser.id, - bot: !$scope.selectedBot ? null : $scope.selectedBot.name, - }, function () { - $scope.bots = botFullDetailsForResultsExistFactory.query({ - browser: !$scope.selectedBrowser ? 'all' : $scope.selectedBrowser.id + $scope.tests_query_on_browser = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + bot: !$scope.selectedBot ? undefined : $scope.selectedBot.name, + }); + $scope.tests = testsForBrowserAndBotFactory.query( + $scope.tests_query_on_browser, function () { + $scope.bot_query = angular.extend({}, { + 'browser': !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id }); + $scope.bots = botFullDetailsForResultsExistFactory.query($scope.bot_query); $scope.reload(); }); }; diff --git a/dashboard/static/js/app_controllers/dashboard/dashboard.services.js b/dashboard/static/js/app_controllers/dashboard/dashboard.services.js index c15bd4a..48d216f 100644 --- a/dashboard/static/js/app_controllers/dashboard/dashboard.services.js +++ b/dashboard/static/js/app_controllers/dashboard/dashboard.services.js @@ -14,25 +14,25 @@ app.factory('botReportsRegressionFactory', function($resource) { }); app.factory('browserFactory', function($resource) { - return $resource('/dash/browser_results_exist'); + return $resource('/dash/browsers'); }); app.factory('botFullDetailsForResultsExistFactory', function($resource) { - return $resource('/dash/bot_full_details_for_exist/:browser'); + return $resource('/dash/bot-full-details'); }); app.factory('platformFactory', function($resource) { - return $resource('/dash/platform_results_exist'); + return $resource('/dash/platforms'); }); app.factory('gpuFactory', function($resource) { - return $resource('/dash/gpu_results_exist'); + return $resource('/dash/gpus'); }); app.factory('cpuArchFactory', function($resource) { - return $resource('/dash/cpu_results_exist'); + return $resource('/dash/cpus'); }); app.factory('testsForBrowserAndBotFactory', function ($resource) { - return $resource('/dash/tests_for_browser_bot/:browser/:bot'); + return $resource('/dash/tests'); }); diff --git a/dashboard/static/js/app_controllers/plot/plot.module.js b/dashboard/static/js/app_controllers/plot/plot.module.js index 8c8d03f..fd29dfc 100644 --- a/dashboard/static/js/app_controllers/plot/plot.module.js +++ b/dashboard/static/js/app_controllers/plot/plot.module.js @@ -38,26 +38,28 @@ app.controller( $scope.onBrowserChange = function () { //Update tests - $scope.tests = testsForBrowserAndBotFactory.query({ - browser: !$scope.selectedBrowser ? 'all' - : $scope.selectedBrowser.id, - bot: !$scope.selectedBot ? null : $scope.selectedBot.name - }, function () { + $scope.tests_filter_on_browser = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + bot: !$scope.selectedBot ? undefined : $scope.selectedBot.name + }); + $scope.tests = testsForBrowserAndBotFactory.query( + $scope.tests_filter_on_browser, function () { $scope.selectedTest = $scope.tests[0]; $scope.onTestsChange(); - $scope.bots = botForResultsExistFactory.query({ - browser: !$scope.selectedBrowser ? 'all' - : $scope.selectedBrowser.id + $scope.query_params = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id }); + $scope.bots = botForResultsExistFactory.query($scope.query_params); }); }; $scope.onBotsChange = function () { - $scope.tests = testsForBrowserAndBotFactory.query({ - browser: !$scope.selectedBrowser ? 'all' - : $scope.selectedBrowser.id, - bot: !$scope.selectedBot ? null : $scope.selectedBot.name - }, function (data) { + $scope.tests_on_bot_change = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + bot: !$scope.selectedBot ? undefined : $scope.selectedBot.name + }); + $scope.tests = testsForBrowserAndBotFactory.query( + $scope.tests_on_bot_change, function (data) { if(data.length === 0) { $scope.selectedTest = []; $scope.selectedSubtest = []; @@ -76,10 +78,12 @@ app.controller( if(!$scope.selectedTest) { return; } - $scope.subtests = subTestPathFactory.query({ - browser: !$scope.selectedBrowser ? 'all' : $scope.selectedBrowser.id, - root_test: $scope.selectedTest.root_test.id - }, function (data) { + $scope.subtests_query = angular.extend({}, { + browser: !$scope.selectedBrowser ? undefined : $scope.selectedBrowser.id, + root_test: !$scope.selectedTest.root_test ? undefined : $scope.selectedTest.root_test.id + }); + $scope.subtests = subTestPathFactory.query( + $scope.subtests_query, function (data) { // We might have something here due to URL data if (!$scope.selectedSubtest) { $scope.selectedSubtest = data[0]; } }); @@ -107,10 +111,7 @@ app.controller( $scope.bots = botForResultsExistFactory.query({ browser: 'all' }); - $scope.tests = testsForBrowserAndBotFactory.query({ - browser: 'all', - bot: null - }); + $scope.tests = testsForBrowserAndBotFactory.query(); //Lets just wait for everything to load, and then populate things $scope.browsers.$promise.then(function () { @@ -126,11 +127,11 @@ app.controller( for ( var i=0; i< plotArray.length; i++ ) { var value = plotArray[i]; - var subtests = subTestPathFactory.query({ - browser: value['browser'], - root_test: value['root_test'] + $scope.subtests_query_laod = angular.extend({},{ + browser: !value['browser'] ? undefined : value['browser'], + root_test: !value['root_test'] ? undefined : value['root_test'] }); - + var subtests = subTestPathFactory.query($scope.subtests_query_laod); $scope.drawGraph( value['browser'], value['bot'], value['root_test'], value['subtest'], value['seq'], value['start'], @@ -200,15 +201,17 @@ app.controller( testMetricsOfTestAndSubtestFactory.query({ root_test: selectedTest.root_test.id, - subtest: encodeURIComponent(selectedSubtest.test_path), + subtest: encodeURIComponent(selectedSubtest.test_path) }, function (testMetrics) { $scope.loading = true; - testResultsForTestAndSubtestFactory.query({ - browser: !selectedBrowser ? 'all' : selectedBrowser.id, - root_test: selectedTest.root_test.id, - bot: !selectedBot ? 'all' : selectedBot.name, + $scope.results_query = angular.extend({}, { + browser: !selectedBrowser ? undefined : selectedBrowser.id, + test: !selectedTest.root_test ? undefined : selectedTest.root_test.id, + bot: !selectedBot ? undefined : selectedBot.name, subtest: encodeURIComponent(selectedSubtest.test_path) - }, function (results) { + }); + testResultsForTestAndSubtestFactory.query( + $scope.results_query, function (results){ if (seq !== undefined) { // This is a draw from the URL - lets use the original sequence for this graph $scope.graphCounter = seq; diff --git a/dashboard/static/js/app_controllers/plot/plot.services.js b/dashboard/static/js/app_controllers/plot/plot.services.js index 00cb89d..10da503 100644 --- a/dashboard/static/js/app_controllers/plot/plot.services.js +++ b/dashboard/static/js/app_controllers/plot/plot.services.js @@ -6,25 +6,25 @@ app = angular.module('browserperfdash.plot.services', ['ngResource']); app.factory('browserForResultExistFactory', function ($resource) { - return $resource('/dash/browser_results_exist'); + return $resource('/dash/browsers'); }); app.factory('botForResultsExistFactory', function($resource) { - return $resource('/dash/bot_results_exist/:browser'); + return $resource('/dash/bots'); }); app.factory('testsForBrowserAndBotFactory', function ($resource) { - return $resource('/dash/tests_for_browser_bot/:browser/:bot'); + return $resource('/dash/tests'); }); app.factory('subTestPathFactory', function ($resource) { - return $resource('/dash/testpath/:browser/:root_test'); + return $resource('/dash/test-paths'); }); app.factory('testMetricsOfTestAndSubtestFactory', function ($resource) { - return $resource('/dash/test_metrics/:root_test/:subtest'); + return $resource('/dash/test-metrics/:root_test/:subtest'); }); app.factory('testResultsForTestAndSubtestFactory', function ($resource) { - return $resource('/dash/results_for_subtest/:browser/:root_test/:bot/:subtest/'); + return $resource('/dash/results'); }); diff --git a/dashboard/urls.py b/dashboard/urls.py index d710d9e..fd4ce0c 100644 --- a/dashboard/urls.py +++ b/dashboard/urls.py @@ -10,8 +10,8 @@ from dashboard.core.bots.views import BotsForResultsExistList, \ BotsFullDetailsForResultsExistList from dashboard.core.bots.reports.views import TestsForBrowserBotList, \ - ResultsForSubtestList, BotDataReportImprovementListView, \ - BotDataReportRegressionListView, TestPathList + ResultsForSubTestList, BotDataReportImprovementListView, \ + BotDataReportRegressionListView, TestPathsList urlpatterns = [ url( @@ -19,17 +19,17 @@ name='graph_report' ), url(r'^bot-report', BotReportView.as_view()), - url(r'^gpu_results_exist/$', GPUTypeForWhichResultsExistList.as_view()), - url(r'^cpu_results_exist/$', CPUArchitectureForWhichResultsExistList.as_view()), - url(r'^platform_results_exist/$', PlatformForWhichResultsExistList.as_view()), - url(r'^browser_results_exist/$', BrowsersForResultsExistList.as_view()), - url(r'^bot_results_exist/(?P.+)$', BotsForResultsExistList.as_view()), - url(r'^bot_full_details_for_exist/(?P.+)$', BotsFullDetailsForResultsExistList.as_view()), - url(r'^testpath/(?P.+)/(?P.+)$', TestPathList.as_view()), - url(r'^test_metrics/(?P[-\w]+)/(?P.+)$', MetricsForTestList.as_view()), - url(r'^tests_for_browser_bot/(?P.+)/(?P.*)$', TestsForBrowserBotList.as_view()), - url(r'^results_for_subtest/(?P[-\w]+)/(?P[-\w]+)/(?P[-\w]+)/(?P.+)/$', - ResultsForSubtestList.as_view()), + url(r'^gpus$', GPUTypeForWhichResultsExistList.as_view()), + url(r'^cpus$', CPUArchitectureForWhichResultsExistList.as_view()), + url(r'^platforms$', PlatformForWhichResultsExistList.as_view()), + url(r'^browsers$', BrowsersForResultsExistList.as_view()), + url(r'^bots$', BotsForResultsExistList.as_view()), + url(r'^bot-full-details$', BotsFullDetailsForResultsExistList.as_view()), + url(r'^test-paths', TestPathsList.as_view()), + url(r'^test-metrics/(?P[-\w]+)/(?P.+)$', + MetricsForTestList.as_view()), + url(r'^tests$', TestsForBrowserBotList.as_view()), + url(r'^results$', ResultsForSubTestList.as_view()), url(r'^report/improvements', BotDataReportImprovementListView.as_view()), url(r'^report/regressions', BotDataReportRegressionListView.as_view()) ] diff --git a/docs/local-settings.py b/docs/local-settings.py index 1e41009..5d0f6a6 100644 --- a/docs/local-settings.py +++ b/docs/local-settings.py @@ -29,5 +29,3 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, "static") - -