Merge pull request #5182 from akvo/master-rebase

Merge production into master

README.md

@@ -11,6 +11,7 @@ We believe that Akvo RSR can be used in many other scenarios, including environm
 
 For more information on Akvo RSR, please visit the [wiki page](https://github.com/akvo/akvo-rsr/wiki). Or use these links to go directly to one of the corresponding pages:
 * [Quick install guide](README.dev.md)
+* [Deployment](doc/deploying.md)
 * [Managing Training Environments](ci/training-envs/README.md)
 * [Release notes](https://github.com/akvo/akvo-rsr/releases)
 * [What's next for RSR](https://github.com/akvo/akvo-rsr/wiki/What's-next-for-RSR)

akvo/rest/serializers/__init__.py

@@ -31,6 +31,7 @@
                         IndicatorFrameworkNotSoLiteSerializer)
 from .indicator_label import IndicatorLabelSerializer
 from .indicator_period import IndicatorPeriodSerializer, IndicatorPeriodFrameworkSerializer
+from .indicator_period_aggregation_job import IndicatorPeriodAggregationJobSerializer
 from .indicator_period_data import (IndicatorPeriodDataSerializer,
                                     IndicatorPeriodDataFrameworkSerializer,
                                     IndicatorPeriodDataCommentSerializer)
@@ -144,6 +145,7 @@
     'IndicatorFrameworkSerializer',
     'IndicatorLabelSerializer',
     'IndicatorPeriodActualLocationSerializer',
+    'IndicatorPeriodAggregationJobSerializer',
     'IndicatorPeriodDataCommentSerializer',
     'IndicatorPeriodDataFrameworkSerializer',
     'IndicatorPeriodDataSerializer',

akvo/rest/serializers/indicator_period_aggregation_job.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+
+from akvo.rest.serializers.rsr_serializer import BaseRSRSerializer
+from akvo.rsr.models import IndicatorPeriodAggregationJob
+
+
+class IndicatorPeriodAggregationJobSerializer(BaseRSRSerializer):
+    class Meta:
+        model = IndicatorPeriodAggregationJob
+        fields = "__all__"

akvo/rest/urls.py

@@ -37,6 +37,7 @@
 router.register(r'(?P<version>(v1))/iati_check', views.IatiCheckViewSet)
 router.register(r'(?P<version>(v1))/iati_export', views.IatiExportViewSet)
 router.register(r'(?P<version>(v1))/indicator', views.IndicatorViewSet)
+router.register(r'(?P<version>(v1))/jobs/indicator_period_aggregation', views.IndicatorPeriodAggregationJobViewSet)
 router.register(r'(?P<version>(v1))/dimension_name', views.IndicatorDimensionNameViewSet)
 router.register(r'(?P<version>(v1))/dimension_value', views.IndicatorDimensionValueViewSet)
 router.register(r'(?P<version>(v1))/indicator_custom_field', views.IndicatorCustomFieldViewSet)

akvo/rest/views/__init__.py

@@ -31,6 +31,7 @@
 from .indicator_dimension_name import IndicatorDimensionNameViewSet
 from .indicator_dimension_value import IndicatorDimensionValueViewSet
 from .indicator_label import IndicatorLabelViewSet
+from .indicator_period_aggregation_job import IndicatorPeriodAggregationJobViewSet
 from .indicator_period_label import IndicatorPeriodLabelViewSet, project_period_labels
 from .indicator_period import (IndicatorPeriodViewSet, IndicatorPeriodFrameworkViewSet,
                                set_periods_locked, bulk_add_periods, bulk_remove_periods)
@@ -139,6 +140,7 @@
     'IatiCheckViewSet',
     'IatiExportViewSet',
     'IndicatorViewSet',
+    'IndicatorPeriodAggregationJobViewSet',
     'IndicatorCustomFieldViewSet',
     'IndicatorCustomValueViewSet',
     'IndicatorDimensionNameViewSet',

akvo/rest/views/indicator_period_aggregation_job.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+# Akvo RSR is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+from rest_framework import filters
+from rest_framework.decorators import action
+from rest_framework.exceptions import ValidationError
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+
+from akvo.rsr.models import IndicatorPeriodAggregationJob
+from ..filters import RSRGenericFilterBackend
+
+from ..serializers import IndicatorPeriodAggregationJobSerializer
+from ..viewsets import ReadOnlyPublicProjectViewSet, SafeMethodsPermissions
+from ...rsr.usecases.jobs.aggregation import schedule_aggregation_job
+
+
+class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet):
+    queryset = IndicatorPeriodAggregationJob.objects.all().select_related(
+        IndicatorPeriodAggregationJob.project_relation[:-2]
+    )
+    serializer_class = IndicatorPeriodAggregationJobSerializer
+    project_relation = IndicatorPeriodAggregationJob.project_relation
+    ordering = ["updated_at"]
+    filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,)
+
+    # These are login only resources that shouldn't be interesting to the public
+    permission_classes = (SafeMethodsPermissions, IsAuthenticated)
+
+    @action(detail=True, methods=['post'])
+    def reschedule(self, request, **kwargs):
+        """
+        Puts a new job in the queue for the indicator period
+
+        The old job is left in order to have a history
+        """
+        job: IndicatorPeriodAggregationJob = self.get_object()
+
+        if job.status != IndicatorPeriodAggregationJob.Status.MAXXED:
+            raise ValidationError("Maximum number of attempts not reached")
+
+        new_job = schedule_aggregation_job(job.period)
+        serializer = self.get_serializer(new_job)
+
+        return Response(serializer.data)

akvo/rest/views/project_overview.py

@@ -15,7 +15,7 @@
 from datetime import date
 from decimal import Decimal, InvalidOperation
 from django.conf import settings
-from django.db.models import Sum
+from django.db.models import Count, Sum
 from django.shortcuts import get_object_or_404
 from functools import cached_property, lru_cache
 from rest_framework.authentication import SessionAuthentication
@@ -465,6 +465,9 @@ def _get_indicator_periods_hierarchy_flatlist(indicator):
         'disaggregation_targets',
         'disaggregation_targets__dimension_value',
         'disaggregation_targets__dimension_value__name'
+    ).annotate(
+        num_aggregation_jobs=Count("aggregation_jobs"),
+        num_child_aggregation_jobs=Count("child_aggregation_jobs"),
     ).filter(pk__in=family)
 
     return periods
@@ -500,7 +503,11 @@ def _transform_period_contributions_node(node, aggregate_targets=False):
     is_qualitative = period.indicator.type == QUALITATIVE
     actual_numerator, actual_denominator = None, None
     updates_value, updates_numerator, updates_denominator = None, None, None
-    contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(node['children'], is_percentage)
+    contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(
+        node['children'],
+        is_percentage,
+        node['item'].num_child_aggregation_jobs,
+    )
     aggregated_value, aggregated_numerator, aggregated_denominator = aggregates
     updates = _transform_updates(period)
 
@@ -558,15 +565,15 @@ def _aggregate_targets(node):
     return aggregate
 
 
-def _transform_contributions_hierarchy(tree, is_percentage):
+def _transform_contributions_hierarchy(tree, is_percentage, root_has_aggregation_job):
     contributors = []
     contributor_countries = []
     aggregated_value = Decimal(0) if not is_percentage else None
     aggregated_numerator = Decimal(0) if is_percentage else None
     aggregated_denominator = Decimal(0) if is_percentage else None
     disaggregations = {}
     for node in tree:
-        contributor, countries = _transform_contributor_node(node, is_percentage)
+        contributor, countries = _transform_contributor_node(node, is_percentage, root_has_aggregation_job)
         if contributor:
             contributors.append(contributor)
             contributor_countries = _merge_unique(contributor_countries, countries)
@@ -616,8 +623,8 @@ def _extract_percentage_updates(updates):
     return numerator, denominator
 
 
-def _transform_contributor_node(node, is_percentage):
-    contributor, aggregate_children = _transform_contributor(node['item'], is_percentage)
+def _transform_contributor_node(node, is_percentage, root_has_aggregation_job):
+    contributor, aggregate_children = _transform_contributor(node['item'], is_percentage, root_has_aggregation_job)
     if not contributor:
         return contributor, []
 
@@ -633,7 +640,9 @@ def _transform_contributor_node(node, is_percentage):
     if not aggregate_children:
         return contributor, contributor_countries
 
-    contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(node['children'], is_percentage)
+    contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(
+        node['children'], is_percentage, node['item'].num_child_aggregation_jobs or root_has_aggregation_job
+    )
     aggregated_value, aggregated_numerator, aggregated_denominator = aggregates
     contributors_count = len(contributors)
     if contributors_count:
@@ -657,14 +666,14 @@ def _calculate_update_values(updates):
     return total
 
 
-def _transform_contributor(period, is_percentage):
+def _transform_contributor(period, is_percentage, root_has_aggregation_job):
     value = _force_decimal(period.actual_value)
 
     is_qualitative = period.indicator.type == QUALITATIVE
     # FIXME: Not sure why the value < 1 check is being used, if it is a float
     # comparison issue, we need to resolve it in a better fashion.
     # Return early if there are not updates and value is "0" for quantitative updates
-    if not is_qualitative and value < 1 and period.data.count() < 1:
+    if not root_has_aggregation_job and not is_qualitative and value < 1 and period.data.count() < 1:
         return None, None
 
     project = period.indicator.result.project

akvo/rsr/admin.py

@@ -854,6 +854,41 @@ def get_queryset(self, request):
 admin.site.register(apps.get_model('rsr', 'IndicatorPeriodData'), IndicatorPeriodDataAdmin)
 
 
+class IndicatorPeriodAggregationJobAdmin(admin.ModelAdmin):
+    model = apps.get_model('rsr', 'IndicatorPeriodAggregationJob')
+    list_display = ('indicator_title', 'status', 'project_title', 'root_project_title', 'period', 'updated_at')
+    list_filter = ('status', )
+    search_fields = ('period__indicator__result__period__title', 'period__indicator__title')
+    readonly_fields = ('updated_at', 'period', 'root_period', 'project_title', 'root_project_title', 'indicator_title')
+
+    @admin.display(description='Project Title')
+    def project_title(self, obj):
+        return self.get_project(obj.period).title
+
+    @admin.display(description='Root project Title')
+    def root_project_title(self, obj):
+        return self.get_project(obj.root_period).title
+
+    def get_project(self, period):
+        return period.indicator.result.project
+
+    @admin.display(description='Indicator Title')
+    def indicator_title(self, obj):
+        return obj.period.indicator.title
+
+    def get_queryset(self, request):
+        queryset = super().get_queryset(request).select_related("period__indicator")
+        if request.user.is_admin or request.user.is_superuser:
+            return queryset
+
+        employments = request.user.approved_employments(['Admins', 'M&E Managers'])
+        projects = employments.organisations().all_projects()
+        return self.model.objects.filter(period__indicator__result__project__in=projects)
+
+
+admin.site.register(apps.get_model('rsr', 'IndicatorPeriodAggregationJob'), IndicatorPeriodAggregationJobAdmin)
+
+
 class ReportAdminForm(forms.ModelForm):
     class Meta:
         model = apps.get_model('rsr', 'Report')

akvo/rsr/dir/app/modules/index/view.jsx

@@ -11,6 +11,7 @@ import Map, { projectsToFeatureData } from './map'
 import Search from './search'
 import FilterBar from './filter-bar'
 import api from '../../utils/api'
+import { isPartnerSites } from '../../utils/misc'
 
 const isLocal = window.location.href.indexOf('localhost') !== -1 || window.location.href.indexOf('localakvoapp') !== -1
 const urlPrefix = isLocal ? 'http://rsr.akvo.org' : ''
@@ -83,7 +84,10 @@ const View = () => {
         setFilters(defaults)
       }
     }
-  }, [apiData])
+    if ((apiError && apiError.response && apiError.response.status === 403) && isPartnerSites()) {
+      window.location.href = '/en/lockpass/?next='
+    }
+  }, [apiData, apiError])
   useEffect(() => {
     document.getElementById('root').classList.add(window.location.host.split('.')[0])
   }, [])

akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx

@@ -1,4 +1,4 @@
-/* global document */
+/* global window, document */
 import React, { useEffect, useState } from 'react'
 import { Menu } from 'antd'
 import { Switch, Route, useHistory } from 'react-router-dom'
@@ -28,6 +28,7 @@ import {
   UPDATES_KEY,
   projectPath,
 } from '../../utils/config'
+import { isPartnerSites } from '../../utils/misc'
 
 
 const ProjectPage = ({ match: { params }, location }) => {
@@ -84,6 +85,9 @@ const ProjectPage = ({ match: { params }, location }) => {
     }
     if ((loading && (apiError || projectError)) || (loading && user && !apiError)) {
       setLoading(false)
+      if ((projectError && projectError.response && projectError.response.status === 403) && isPartnerSites()) {
+        window.location.href = `/en/lockpass/?next=${window.location.href}`
+      }
     }
     // eslint-disable-next-line no-restricted-globals
     if (!isNaN(currentPath) && menu !== HOME_KEY) {

akvo/rsr/dir/app/root.jsx

@@ -8,12 +8,13 @@ import WcaroRouter from './modules/wcaro-index/router'
 import ProjectPage from './modules/project-page/ProjectPage'
 import scheduleDemo from './modules/schedule-demo'
 import { Home } from './modules/home'
+import { isPartnerSites } from './utils/misc'
 
 export default () => {
     const isUNEP = window.location.href.indexOf('//unep.') !== -1
     // eslint-disable-next-line no-unused-vars
     const isWcaro = window.location.href.indexOf('//wcaro.') !== -1
-    const isPartner = window.location.href.indexOf('localakvoapp') !== -1 || window.location.href.indexOf('akvoapp') !== -1
+    const isPartner = isPartnerSites()
     return (
         <Router basename="/">
             <Route path="/" exact component={isUNEP ? UnepIndex : isPartner ? Index : Home} />

akvo/rsr/dir/app/utils/misc.js

@@ -1,4 +1,5 @@
 /* eslint-disable no-useless-escape */
+/* global window */
 import moment from 'moment'
 import chunk from 'lodash/chunk'
 import orderBy from 'lodash/orderBy'
@@ -99,3 +100,5 @@ export const getLogo = logo => logo
 export const getFirstPhoto = photos => (photos && photos.length)
   ? photos.slice(0, 1).pop()
   : null
+
+export const isPartnerSites = () => (window.location.hostname.endsWith('localakvoapp.org') || window.location.hostname.endsWith('akvoapp.org'))

akvo/rsr/management/commands/link_results_framework_to_parent_project.py

@@ -7,6 +7,7 @@
 from django.core.management.base import BaseCommand
 
 from akvo.rsr.models import Project
+from akvo.rsr.usecases.period_update_aggregation import aggregate
 
 
 class Command(BaseCommand):
@@ -35,4 +36,4 @@ def handle(self, *args, **options):
                     parent_period = parent_indicator.periods.get(period_start=period.period_start, period_end=period.period_end)
                     period.parent_period = parent_period
                     period.save(update_fields=['parent_period'])
-                    parent_period.recalculate_period()
+                    aggregate(parent_period)

akvo/rsr/management/commands/recalculate_periods.py

@@ -7,6 +7,7 @@
 
 from django.core.management.base import BaseCommand
 from django.db.models import Count
+from akvo.rsr.usecases.period_update_aggregation import aggregate
 
 from ...models import IndicatorPeriod
 
@@ -41,4 +42,4 @@ def handle(self, *args, **options):
         for period in periods:
             if verbosity > 1:
                 print('Recalculating period {}'.format(period.id))
-            period.recalculate_period()
+            aggregate(period)

akvo/rsr/management/commands/recalculate_program_aggregation.py

@@ -0,0 +1,30 @@
+from django.core.management.base import BaseCommand
+from django.db.models import Count, Q
+
+from akvo.rsr.models import ProjectHierarchy, IndicatorPeriod, IndicatorPeriodData
+from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job
+
+
+class Command(BaseCommand):
+    help = 'Script for recalculating periods aggregation of a program'
+
+    def add_arguments(self, parser):
+        parser.add_argument('program_id', type=int)
+
+    def handle(self, *args, **options):
+        try:
+            hierarchy = ProjectHierarchy.objects.get(root_project=options['program_id'])
+            program = hierarchy.root_project
+        except ProjectHierarchy.DoesNotExist:
+            print("Program not found")
+            return
+
+        descendants = program.descendants()
+        periods = IndicatorPeriod.objects\
+            .annotate(approved_count=Count('data', filter=Q(data__status=IndicatorPeriodData.STATUS_APPROVED_CODE)))\
+            .filter(approved_count__gte=1, indicator__result__project__in=descendants)
+
+        for period in periods:
+            schedule_aggregation_job(period)
+
+        print(f"Scheduled period aggregation jobs: {periods.count()}, on program: {program.title}")

akvo/rsr/management/commands/run_aggregation_jobs.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from akvo.rsr.usecases.jobs.aggregation import execute_aggregation_jobs
+
+
+class Command(BaseCommand):
+    help = "Run indicator period aggregation jobs."
+
+    def handle(self, *args, **options):
+        execute_aggregation_jobs()