From 1fe0e7f020bb48d609b0eaedf9988535ac3e55a1 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 27 Sep 2022 10:58:54 +0200 Subject: [PATCH 01/59] Add documentation for deployment --- README.md | 1 + ci/deploy-semaphoreci.sh | 2 +- doc/deploying.md | 74 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 doc/deploying.md diff --git a/README.md b/README.md index 5f35e86a91..b7f99606e8 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ We believe that Akvo RSR can be used in many other scenarios, including environm For more information on Akvo RSR, please visit the [wiki page](https://github.com/akvo/akvo-rsr/wiki). Or use these links to go directly to one of the corresponding pages: * [Quick install guide](README.dev.md) +* [Deployment](doc/deploying.md) * [Managing Training Environments](ci/training-envs/README.md) * [Release notes](https://github.com/akvo/akvo-rsr/releases) * [What's next for RSR](https://github.com/akvo/akvo-rsr/wiki/What's-next-for-RSR) diff --git a/ci/deploy-semaphoreci.sh b/ci/deploy-semaphoreci.sh index 51085efc66..a57d8354b0 100755 --- a/ci/deploy-semaphoreci.sh +++ b/ci/deploy-semaphoreci.sh @@ -29,7 +29,7 @@ if [[ "${CI_TAG:-}" =~ promote-.* ]]; then gcloud container clusters get-credentials production K8S_CONFIG_FILE=ci/k8s/config-prod.yml else - log Environement is test + log Environment is test gcloud container clusters get-credentials test K8S_CONFIG_FILE=ci/k8s/config-test.yml diff --git a/doc/deploying.md b/doc/deploying.md new file mode 100644 index 0000000000..8d07549210 --- /dev/null +++ b/doc/deploying.md @@ -0,0 +1,74 @@ +# Deploying + +We use SemaphoreCI to deploy to test and production + +# Test + +Deploying to https://rsr.akvotest.org is done automatically once a PR is merged to master. + + +# Production + +In order to deploy to [production](https://rsr.akvo.org), the docker image must exist in the docker registry. +At the moment, this is all tied to Akvo's private, docker registry on Google Cloud. + +The docker image is created during the deployment to the test environment, so that **must** have been done first. + + +```mermaid +sequenceDiagram + actor dev as developer + participant local as local system + participant github as Github + participant semaphore as semaphoreCI + participant kube as Google Kubernetes + participant zulip + + dev ->> local: checkout production branch + + dev ->> local: ci/promote-test-to-prod.sh + activate local + local ->> kube: get running versions + activate kube + kube -->> local: versions of TEST & PROD + deactivate kube + local ->> local: print github diff link + local -->> dev: commands for git tag and zulip notif + deactivate local + + dev ->> local: git tag + + dev ->> local: git push + activate local + local -->> github: tag + activate github + github ->> local: + local ->> dev: + deactivate local + + dev ->> local: notify through zulip + activate local + local ->> zulip: notify others + activate zulip + zulip ->> local: notified + deactivate zulip + local ->> dev: + deactivate local + + + github ->> semaphore: new tag pushed + activate semaphore + semaphore -->> github: OK + deactivate github + semaphore ->> kube: deploy tagged version + activate kube + kube ->> semaphore: applying new config + deactivate kube + loop Until containers are ready + semaphore ->> kube: please provide container status + activate kube + kube -->> semaphore: status + deactivate kube + end + deactivate semaphore +``` \ No newline at end of file From d53b5c537b585d24d1e5c254bdd6ae011e67f91d Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 27 Sep 2022 16:21:48 +0200 Subject: [PATCH 02/59] feat: Support promoting production branch to prod While "Improve project hierarchies #4710" is being worked on and merged into master, the production branch will serve as the branch to be promoted to production. The master branch will continuously be deployed to the test environment, while the production branch can be promoted to a test environment. --- ci/deploy-semaphoreci.sh | 12 ++++++++++-- ci/promote-test-to-prod.sh | 13 +++++++++++-- doc/deploying.md | 21 +++++++++++++++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/ci/deploy-semaphoreci.sh b/ci/deploy-semaphoreci.sh index a57d8354b0..bdf0f2b4c6 100755 --- a/ci/deploy-semaphoreci.sh +++ b/ci/deploy-semaphoreci.sh @@ -9,7 +9,15 @@ function log { log Running deployment script export PROJECT_NAME=akvo-lumen -if [[ "${CI_BRANCH}" != "master" ]] && [[ "${CI_BRANCH}" != rsr-env-* ]] && [[ ! "${CI_TAG:-}" =~ promote-.* ]]; then +# rsr-env-* tag builds an image for later deployment in training-env +# production branch builds and image for later deployment in production +# master applies config in test +# promote-* tags applies config in production +if [[ "${CI_BRANCH}" != "master" ]] \ + && [[ "${CI_BRANCH}" != "production" ]] \ + && [[ "${CI_BRANCH}" != rsr-env-* ]] \ + && [[ ! "${CI_TAG:-}" =~ promote-.* ]] +then exit 0 fi @@ -44,7 +52,7 @@ else fi -if [[ "${CI_BRANCH}" = rsr-env-* ]]; then +if [[ "${CI_BRANCH}" = rsr-env-* ]] || [[ "${CI_BRANCH}" = "production" ]]; then exit 0 fi diff --git a/ci/promote-test-to-prod.sh b/ci/promote-test-to-prod.sh index fd8ecd1724..c5b3c3e799 100755 --- a/ci/promote-test-to-prod.sh +++ b/ci/promote-test-to-prod.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -eu -git fetch +DEPLOY_COMMITISH="${1:-production}" +git fetch --all echo "Running E2E tests" if [[ $(uname -s) == "Linux" ]] && [[ ! "${HEADLESS:-}" = false ]]; then @@ -10,5 +11,13 @@ else ./scripts/devhelpers/run-e2e-local.sh fi +COMMIT_HASH="$(git rev-parse ${DEPLOY_COMMITISH})" + docker run --rm -e ZULIP_CLI_TOKEN -v ~/.config:/home/akvo/.config -v "$(pwd)":/app \ - -it akvo/akvo-devops:20201203.085214.79bec73 promote-test-to-prod.sh rsr rsr-version akvo-rsr zulip "Akvo RSR Engine" + -it akvo/akvo-devops:20220927.141344.aa0697b promote-test-to-prod.sh \ + rsr \ + rsr-version \ + akvo-rsr \ + zulip \ + "Akvo RSR Engine" \ + "$COMMIT_HASH" diff --git a/doc/deploying.md b/doc/deploying.md index 8d07549210..c8ec79d901 100644 --- a/doc/deploying.md +++ b/doc/deploying.md @@ -9,10 +9,27 @@ Deploying to https://rsr.akvotest.org is done automatically once a PR is merged # Production +Currently, we use the `production` branch to promote changes to master. +This a **manual** operation. + In order to deploy to [production](https://rsr.akvo.org), the docker image must exist in the docker registry. At the moment, this is all tied to Akvo's private, docker registry on Google Cloud. -The docker image is created during the deployment to the test environment, so that **must** have been done first. +The docker image is created during the deployment to the test environment or simply by merging into `production`. + +## Commands + +**Requirements** + + - A [token from zulip](https://akvo.zulipchat.com/#settings/account-and-privacy) to send messages to the channel. + - Access to Google Cloud (ask somebody from the DevOps team) + +```shell +export ZULIP_TOKEN="YOURTOKENHERE" +ci/promote-test-to-prod.sh +``` + +## Sequence diagram ```mermaid @@ -71,4 +88,4 @@ sequenceDiagram deactivate kube end deactivate semaphore -``` \ No newline at end of file +``` From 1f7e3a3a910dc57816ed1f3e343aba856ba58623 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Thu, 6 Oct 2022 17:07:55 +0700 Subject: [PATCH 03/59] [#5133] Fix Project found incorrect programme Change initial index from 2 to 1 because in the section 1 still has a relatedproject endpoint that need to call. and the bug related with commit [fdb5224c0a1b11b2cc481bb85c687347765aa9b6](https://github.com/akvo/akvo-rsr/commit/fdb5224c0a1b11b2cc481bb85c687347765aa9b6) when I was tried to clean up fetching process in one place. --- akvo/rsr/spa/app/modules/editor/project-init-handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akvo/rsr/spa/app/modules/editor/project-init-handler.js b/akvo/rsr/spa/app/modules/editor/project-init-handler.js index 8377393209..7ef71c9e7f 100644 --- a/akvo/rsr/spa/app/modules/editor/project-init-handler.js +++ b/akvo/rsr/spa/app/modules/editor/project-init-handler.js @@ -46,7 +46,7 @@ const sectionHasEndpoint = [ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { const [preload, setPreload] = useState(true) - const [sectionIndex, setNextSectionIndex] = useState(2) // I started with 2 because 1 will be called first + const [sectionIndex, setNextSectionIndex] = useState(1) const fetchSectionOne = endpoint => api .get(insertRouteParams(endpoint, { projectId: params.id })) From 513b77cd5ee51ae838694698cbb4f91c5a420085 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 11 Oct 2022 17:03:43 +0200 Subject: [PATCH 04/59] feat: Add aggregation job model A basic model to track aggregation jobs. Usecases will come later #5140: Stabilize period update aggregations --- akvo/rsr/models/__init__.py | 2 + akvo/rsr/models/aggregation_job.py | 7 + akvo/rsr/models/cron_job.py | 45 +++++++ akvo/rsr/tests/usecases/jobs/__init__.py | 0 .../tests/usecases/jobs/test_aggregation.py | 120 ++++++++++++++++++ akvo/rsr/usecases/jobs/__init__.py | 0 akvo/rsr/usecases/jobs/aggregation.py | 28 ++++ akvo/rsr/usecases/jobs/cron.py | 9 ++ scripts/deployment/pip/requirements/2_rsr.txt | 34 +++++ scripts/deployment/pip/requirements/3_dev.txt | 34 +++++ .../deployment/pip/requirements/production.in | 1 + 11 files changed, 280 insertions(+) create mode 100644 akvo/rsr/models/aggregation_job.py create mode 100644 akvo/rsr/models/cron_job.py create mode 100644 akvo/rsr/tests/usecases/jobs/__init__.py create mode 100644 akvo/rsr/tests/usecases/jobs/test_aggregation.py create mode 100644 akvo/rsr/usecases/jobs/__init__.py create mode 100644 akvo/rsr/usecases/jobs/aggregation.py create mode 100644 akvo/rsr/usecases/jobs/cron.py diff --git a/akvo/rsr/models/__init__.py b/akvo/rsr/models/__init__.py index bc16fc0d4a..eb0186f512 100644 --- a/akvo/rsr/models/__init__.py +++ b/akvo/rsr/models/__init__.py @@ -18,6 +18,7 @@ update_project_budget, update_project_funding ) +from .aggregation_job import PeriodUpdateAggregationJob from .benchmark import Benchmark, Benchmarkname from .budget_item import BudgetItem, BudgetItemLabel, CountryBudgetItem from .country import Country, RecipientCountry @@ -170,6 +171,7 @@ 'Partnership', 'PeriodActualValue', 'PeriodDisaggregation', + 'PeriodUpdateAggregationJob', 'IndicatorPeriodDisaggregation', 'PlannedDisbursement', 'PolicyMarker', diff --git a/akvo/rsr/models/aggregation_job.py b/akvo/rsr/models/aggregation_job.py new file mode 100644 index 0000000000..5ae93acfed --- /dev/null +++ b/akvo/rsr/models/aggregation_job.py @@ -0,0 +1,7 @@ +from django.db import models + +from akvo.rsr.models.cron_job import CronJobMixin + + +class PeriodUpdateAggregationJob(CronJobMixin): + period_update = models.ForeignKey("IndicatorPeriod", on_delete=models.CASCADE) diff --git a/akvo/rsr/models/cron_job.py b/akvo/rsr/models/cron_job.py new file mode 100644 index 0000000000..9977eb921d --- /dev/null +++ b/akvo/rsr/models/cron_job.py @@ -0,0 +1,45 @@ +import os + +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class CronJobMixin(models.Model): + class Status(models.TextChoices): + SCHEDULED = 'SCHEDULED', _('Scheduled') + RUNNING = 'RUNNING', _('Running') + FINISHED = 'FINISHED', _('Finished') + FAILED = 'FAILED', _('Failed') + + status = models.CharField( + max_length=10, + choices=Status.choices, + default=Status.SCHEDULED, + ) + attempts = models.IntegerField(default=0) + pid = models.PositiveIntegerField(null=True) + updated_at = models.DateTimeField(auto_now=True) + + def mark_running(self): + self.status = self.Status.RUNNING + self.pid = os.getpid() + self.save() + + def mark_finished(self): + self.status = self.Status.FINISHED + self.pid = None + self.attempts = self.attempts + 1 + self.save() + + def mark_scheduled(self): + self.status = self.Status.SCHEDULED + self.pid = None + self.save() + + def mark_failed(self): + self.status = self.Status.FAILED + self.pid = None + self.save() + + class Meta: + abstract = True diff --git a/akvo/rsr/tests/usecases/jobs/__init__.py b/akvo/rsr/tests/usecases/jobs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py new file mode 100644 index 0000000000..0cc21b2b16 --- /dev/null +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -0,0 +1,120 @@ +import datetime + +from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User +from akvo.rsr.models.aggregation_job import PeriodUpdateAggregationJob +from akvo.rsr.tests.base import BaseTestCase +from akvo.rsr.usecases.jobs.aggregation import get_failed_jobs, get_finished_jobs, get_running_jobs, get_scheduled_jobs +from akvo.rsr.usecases.jobs.cron import is_job_dead + + +class AggregationJobBaseTests(BaseTestCase): + + def setUp(self): + super().setUp() + self.user = User.objects.create_superuser( + username="Super user", + email="superuser.results@test.akvo.org", + password="password" + ) + + self.project = self.create_project('Test project') + + # Create results framework + self.result = Result.objects.create( + project=self.project, title='Result #1', type='1' + ) + self.indicator = Indicator.objects.create( + result=self.result, title='Indicator #1' + ) + self.period = IndicatorPeriod.objects.create( + indicator=self.indicator, + period_start=datetime.date.today(), + period_end=datetime.date.today() + datetime.timedelta(days=1), + target_value="100" + ) + self.job = PeriodUpdateAggregationJob.objects.create(period_update=self.period) + + +class ScheduledJobTestCase(AggregationJobBaseTests): + + def test_get_scheduled(self): + self.assertEqual(self.job, get_scheduled_jobs().first()) + + def test_not_scheduled(self): + self.job.mark_running() + self.assertIsNone(get_scheduled_jobs().first()) + + self.job.mark_finished() + self.assertIsNone(get_scheduled_jobs().first()) + + self.job.mark_failed() + self.assertIsNone(get_scheduled_jobs().first()) + + +class RunningJobTestCase(AggregationJobBaseTests): + + def test_is_running(self): + self.job.mark_running() + + self.assertEqual(self.job, get_running_jobs().first()) + self.assertIsNotNone(self.job.pid) + + def test_not_running(self): + self.job.mark_scheduled() + self.assertIsNone(get_running_jobs().first()) + + self.job.mark_finished() + self.assertIsNone(get_running_jobs().first()) + + self.job.mark_failed() + self.assertIsNone(get_running_jobs().first()) + + def test_is_alive(self): + self.job.mark_running() + + self.assertFalse(is_job_dead(self.job)) + + def test_is_dead(self): + self.job.mark_running() + self.job.pid = 1_000_000 + self.job.save() + + self.assertTrue(is_job_dead(self.job)) + + +class FailedJobTestCase(AggregationJobBaseTests): + + def test_is_failed(self): + self.job.mark_failed() + + self.assertEqual(self.job, get_failed_jobs().first()) + self.assertIsNone(self.job.pid) + + def test_not_failed(self): + self.job.mark_scheduled() + self.assertIsNone(get_failed_jobs().first()) + + self.job.mark_running() + self.assertIsNone(get_failed_jobs().first()) + + self.job.mark_finished() + self.assertIsNone(get_failed_jobs().first()) + + +class FinishedJobTestCase(AggregationJobBaseTests): + + def test_is_finished(self): + self.job.mark_finished() + + self.assertEqual(self.job, get_finished_jobs().first()) + self.assertIsNone(self.job.pid) + + def test_not_finished(self): + self.job.mark_scheduled() + self.assertIsNone(get_finished_jobs().first()) + + self.job.mark_running() + self.assertIsNone(get_finished_jobs().first()) + + self.job.mark_failed() + self.assertIsNone(get_finished_jobs().first()) diff --git a/akvo/rsr/usecases/jobs/__init__.py b/akvo/rsr/usecases/jobs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py new file mode 100644 index 0000000000..3ff3b3cf60 --- /dev/null +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -0,0 +1,28 @@ +from django.db.models import QuerySet + +from akvo.rsr.models.aggregation_job import PeriodUpdateAggregationJob +from akvo.rsr.models.cron_job import CronJobMixin + + +def get_scheduled_jobs() -> QuerySet[PeriodUpdateAggregationJob]: + return PeriodUpdateAggregationJob.objects.filter( + status=CronJobMixin.Status.SCHEDULED, + ) + + +def get_running_jobs() -> QuerySet[PeriodUpdateAggregationJob]: + return PeriodUpdateAggregationJob.objects.filter( + status=CronJobMixin.Status.RUNNING, + ) + + +def get_failed_jobs() -> QuerySet[PeriodUpdateAggregationJob]: + return PeriodUpdateAggregationJob.objects.filter( + status=CronJobMixin.Status.FAILED, + ) + + +def get_finished_jobs() -> QuerySet[PeriodUpdateAggregationJob]: + return PeriodUpdateAggregationJob.objects.filter( + status=CronJobMixin.Status.FINISHED, + ) diff --git a/akvo/rsr/usecases/jobs/cron.py b/akvo/rsr/usecases/jobs/cron.py new file mode 100644 index 0000000000..a827d34f32 --- /dev/null +++ b/akvo/rsr/usecases/jobs/cron.py @@ -0,0 +1,9 @@ +import psutil + +from akvo.rsr.models.cron_job import CronJobMixin + + +def is_job_dead(job: CronJobMixin): + if not (pid := job.pid): + return False + return not psutil.pid_exists(pid) and job.status == job.Status.RUNNING diff --git a/scripts/deployment/pip/requirements/2_rsr.txt b/scripts/deployment/pip/requirements/2_rsr.txt index a6b9b91ef6..fdc5892598 100644 --- a/scripts/deployment/pip/requirements/2_rsr.txt +++ b/scripts/deployment/pip/requirements/2_rsr.txt @@ -593,6 +593,40 @@ protobuf==3.15.0 \ # google-api-core # google-cloud-profiler # googleapis-common-protos +psutil==5.9.2 \ + --hash=sha256:14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97 \ + --hash=sha256:256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84 \ + --hash=sha256:39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969 \ + --hash=sha256:404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf \ + --hash=sha256:42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32 \ + --hash=sha256:4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12 \ + --hash=sha256:4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727 \ + --hash=sha256:561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06 \ + --hash=sha256:5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c \ + --hash=sha256:614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9 \ + --hash=sha256:67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea \ + --hash=sha256:68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8 \ + --hash=sha256:7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f \ + --hash=sha256:8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c \ + --hash=sha256:91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d \ + --hash=sha256:94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339 \ + --hash=sha256:9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444 \ + --hash=sha256:b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab \ + --hash=sha256:b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb \ + --hash=sha256:b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b \ + --hash=sha256:b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8 \ + --hash=sha256:b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec \ + --hash=sha256:d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8 \ + --hash=sha256:dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d \ + --hash=sha256:e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34 \ + --hash=sha256:e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85 \ + --hash=sha256:ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1 \ + --hash=sha256:f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9 \ + --hash=sha256:f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1 \ + --hash=sha256:f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d \ + --hash=sha256:fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5 \ + --hash=sha256:feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c + # via -r scripts/deployment/pip/requirements/production.in psycopg2-binary==2.8.4 \ --hash=sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29 \ --hash=sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03 \ diff --git a/scripts/deployment/pip/requirements/3_dev.txt b/scripts/deployment/pip/requirements/3_dev.txt index 98d297af26..a2458278b1 100644 --- a/scripts/deployment/pip/requirements/3_dev.txt +++ b/scripts/deployment/pip/requirements/3_dev.txt @@ -721,6 +721,40 @@ protobuf==3.15.0 \ # google-api-core # google-cloud-profiler # googleapis-common-protos +psutil==5.9.2 \ + --hash=sha256:14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97 \ + --hash=sha256:256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84 \ + --hash=sha256:39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969 \ + --hash=sha256:404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf \ + --hash=sha256:42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32 \ + --hash=sha256:4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12 \ + --hash=sha256:4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727 \ + --hash=sha256:561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06 \ + --hash=sha256:5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c \ + --hash=sha256:614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9 \ + --hash=sha256:67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea \ + --hash=sha256:68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8 \ + --hash=sha256:7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f \ + --hash=sha256:8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c \ + --hash=sha256:91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d \ + --hash=sha256:94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339 \ + --hash=sha256:9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444 \ + --hash=sha256:b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab \ + --hash=sha256:b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb \ + --hash=sha256:b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b \ + --hash=sha256:b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8 \ + --hash=sha256:b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec \ + --hash=sha256:d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8 \ + --hash=sha256:dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d \ + --hash=sha256:e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34 \ + --hash=sha256:e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85 \ + --hash=sha256:ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1 \ + --hash=sha256:f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9 \ + --hash=sha256:f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1 \ + --hash=sha256:f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d \ + --hash=sha256:fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5 \ + --hash=sha256:feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c + # via -r scripts/deployment/pip/requirements/production.in psycopg2-binary==2.8.4 \ --hash=sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29 \ --hash=sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03 \ diff --git a/scripts/deployment/pip/requirements/production.in b/scripts/deployment/pip/requirements/production.in index 2feef39cf5..9f144b5b87 100644 --- a/scripts/deployment/pip/requirements/production.in +++ b/scripts/deployment/pip/requirements/production.in @@ -74,6 +74,7 @@ django-prettyjson # Task scheduling django-crontab +psutil # XML Testing xmlunittest==0.5 From a520f0600183b0013c5e36c64ec88cee5c45a07f Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 11 Oct 2022 17:03:43 +0200 Subject: [PATCH 05/59] refactor: Fix aggregation job model After discussing with colleagues, the specs changed and we had to revert. #5140: Stabilize period update aggregations --- .../0221_indicatorupdateaggregationjob.py | 29 +++++++++++++++++++ akvo/rsr/models/__init__.py | 4 +-- akvo/rsr/models/aggregation_job.py | 5 ++-- akvo/rsr/models/cron_job.py | 9 +++++- .../tests/usecases/jobs/test_aggregation.py | 6 ++-- akvo/rsr/usecases/jobs/aggregation.py | 25 ++++++++++------ 6 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py diff --git a/akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py b/akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py new file mode 100644 index 0000000000..500cd575ac --- /dev/null +++ b/akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.10 on 2022-10-12 12:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('rsr', '0220_emailreportjob'), + ] + + operations = [ + migrations.CreateModel( + name='IndicatorUpdateAggregationJob', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(choices=[('SCHEDULED', 'Scheduled'), ('RUNNING', 'Running'), ('FINISHED', 'Finished'), ('FAILED', 'Failed'), ('dododod', 'Failed')], default='SCHEDULED', max_length=10)), + ('attempts', models.IntegerField(default=0)), + ('pid', models.PositiveIntegerField(null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rsr.indicatorperiod')), + ('program', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rsr.project')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/akvo/rsr/models/__init__.py b/akvo/rsr/models/__init__.py index eb0186f512..bc4a3cf187 100644 --- a/akvo/rsr/models/__init__.py +++ b/akvo/rsr/models/__init__.py @@ -18,7 +18,7 @@ update_project_budget, update_project_funding ) -from .aggregation_job import PeriodUpdateAggregationJob +from .aggregation_job import IndicatorUpdateAggregationJob from .benchmark import Benchmark, Benchmarkname from .budget_item import BudgetItem, BudgetItemLabel, CountryBudgetItem from .country import Country, RecipientCountry @@ -171,7 +171,7 @@ 'Partnership', 'PeriodActualValue', 'PeriodDisaggregation', - 'PeriodUpdateAggregationJob', + 'IndicatorUpdateAggregationJob', 'IndicatorPeriodDisaggregation', 'PlannedDisbursement', 'PolicyMarker', diff --git a/akvo/rsr/models/aggregation_job.py b/akvo/rsr/models/aggregation_job.py index 5ae93acfed..bc7bca3e1c 100644 --- a/akvo/rsr/models/aggregation_job.py +++ b/akvo/rsr/models/aggregation_job.py @@ -3,5 +3,6 @@ from akvo.rsr.models.cron_job import CronJobMixin -class PeriodUpdateAggregationJob(CronJobMixin): - period_update = models.ForeignKey("IndicatorPeriod", on_delete=models.CASCADE) +class IndicatorUpdateAggregationJob(CronJobMixin): + period = models.ForeignKey("IndicatorPeriod", on_delete=models.CASCADE) + program = models.ForeignKey("Project", on_delete=models.CASCADE) diff --git a/akvo/rsr/models/cron_job.py b/akvo/rsr/models/cron_job.py index 9977eb921d..f72adf7745 100644 --- a/akvo/rsr/models/cron_job.py +++ b/akvo/rsr/models/cron_job.py @@ -10,9 +10,10 @@ class Status(models.TextChoices): RUNNING = 'RUNNING', _('Running') FINISHED = 'FINISHED', _('Finished') FAILED = 'FAILED', _('Failed') + MAXXED = 'MAXXED', _('Max attempts reached') status = models.CharField( - max_length=10, + max_length=15, choices=Status.choices, default=Status.SCHEDULED, ) @@ -38,6 +39,12 @@ def mark_scheduled(self): def mark_failed(self): self.status = self.Status.FAILED + self.attempts = self.attempts + 1 + self.pid = None + self.save() + + def mark_max_reached(self): + self.status = self.Status.MAXXED self.pid = None self.save() diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 0cc21b2b16..d4b48ce00f 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -1,7 +1,7 @@ import datetime from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User -from akvo.rsr.models.aggregation_job import PeriodUpdateAggregationJob +from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.usecases.jobs.aggregation import get_failed_jobs, get_finished_jobs, get_running_jobs, get_scheduled_jobs from akvo.rsr.usecases.jobs.cron import is_job_dead @@ -32,7 +32,7 @@ def setUp(self): period_end=datetime.date.today() + datetime.timedelta(days=1), target_value="100" ) - self.job = PeriodUpdateAggregationJob.objects.create(period_update=self.period) + self.job = IndicatorUpdateAggregationJob.objects.create(period=self.period, program=self.project) class ScheduledJobTestCase(AggregationJobBaseTests): @@ -89,6 +89,7 @@ def test_is_failed(self): self.assertEqual(self.job, get_failed_jobs().first()) self.assertIsNone(self.job.pid) + self.assertEqual(self.job.attempts, 1) def test_not_failed(self): self.job.mark_scheduled() @@ -108,6 +109,7 @@ def test_is_finished(self): self.assertEqual(self.job, get_finished_jobs().first()) self.assertIsNone(self.job.pid) + self.assertEqual(self.job.attempts, 1) def test_not_finished(self): self.job.mark_scheduled() diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 3ff3b3cf60..1debe83725 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -1,28 +1,35 @@ from django.db.models import QuerySet -from akvo.rsr.models.aggregation_job import PeriodUpdateAggregationJob +from akvo.rsr.models import IndicatorPeriod +from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob from akvo.rsr.models.cron_job import CronJobMixin -def get_scheduled_jobs() -> QuerySet[PeriodUpdateAggregationJob]: - return PeriodUpdateAggregationJob.objects.filter( +def get_scheduled_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: + return IndicatorUpdateAggregationJob.objects.filter( status=CronJobMixin.Status.SCHEDULED, ) -def get_running_jobs() -> QuerySet[PeriodUpdateAggregationJob]: - return PeriodUpdateAggregationJob.objects.filter( +def get_running_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: + return IndicatorUpdateAggregationJob.objects.filter( status=CronJobMixin.Status.RUNNING, ) -def get_failed_jobs() -> QuerySet[PeriodUpdateAggregationJob]: - return PeriodUpdateAggregationJob.objects.filter( +def get_failed_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: + return IndicatorUpdateAggregationJob.objects.filter( status=CronJobMixin.Status.FAILED, ) -def get_finished_jobs() -> QuerySet[PeriodUpdateAggregationJob]: - return PeriodUpdateAggregationJob.objects.filter( +def get_maxxed_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: + return IndicatorUpdateAggregationJob.objects.filter( + status=CronJobMixin.Status.MAXXED, + ) + + +def get_finished_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: + return IndicatorUpdateAggregationJob.objects.filter( status=CronJobMixin.Status.FINISHED, ) From 8f2ff76bb9ebde1221a62197390f74970bdad858 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 12 Oct 2022 16:36:13 +0200 Subject: [PATCH 06/59] feat: Implement logic for handling aggregation tasks #5140: Stabilize period update aggregations --- akvo/rsr/usecases/jobs/aggregation.py | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 1debe83725..3049456f6a 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -1,8 +1,11 @@ from django.db.models import QuerySet +from django.db.transaction import atomic +from typing import List from akvo.rsr.models import IndicatorPeriod from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob from akvo.rsr.models.cron_job import CronJobMixin +from akvo.rsr.usecases.jobs.cron import is_job_dead def get_scheduled_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: @@ -33,3 +36,66 @@ def get_finished_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: return IndicatorUpdateAggregationJob.objects.filter( status=CronJobMixin.Status.FINISHED, ) + + +def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorUpdateAggregationJob: + """ + Schedule a job for the period to be aggregated upwards if no job exists + """ + if existing_job := get_scheduled_jobs().filter(period=period).first(): + return existing_job + return IndicatorUpdateAggregationJob.objects.create(period=period) + + +def execute_aggregation_jobs(): + """ + Call the aggregation function for each aggregation job + """ + handle_failed_jobs() + + scheduled_jobs = get_scheduled_jobs() + for scheduled_job in scheduled_jobs: + with atomic(): + scheduled_job.mark_running() + try: + run_aggregation(scheduled_job.period) + scheduled_job.mark_finished() + except Exception as e: + scheduled_job.mark_failed() + email_failed_job_owner(scheduled_job, str(e)) + + +@atomic +def handle_failed_jobs(): + """Identify failed jobs, notify owners, and reschedule them""" + identify_dead_jobs() + failed_jobs = get_failed_jobs() + + for failed_job in failed_jobs: + email_failed_job_owner(failed_job, "Job died") + + failed_job.mark_scheduled() + + +@atomic +def identify_dead_jobs() -> List[IndicatorUpdateAggregationJob]: + """ + Find jobs that are supposed to be running, but with a dead process + """ + dead_jobs = [] + for running_job in get_running_jobs(): + if not is_job_dead(running_job): + continue + running_job.mark_failed() + dead_jobs.append(running_job) + + return dead_jobs + + +def email_failed_job_owner(failed_job: IndicatorUpdateAggregationJob, reason: str): + raise NotImplementedError() + + +def run_aggregation(period: IndicatorPeriod): + # TODO: Implement + raise NotImplementedError() From d1c1456b7ae4573bf007eeb6324b1947d1922451 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Thu, 13 Oct 2022 15:42:37 +0700 Subject: [PATCH 07/59] [#5148] Fix bug saving description of a project Replace `isFetched` property with id because that is the field has a strong condition to indicate all fields saved/present in the `editorRdr`. --- akvo/rsr/spa/app/modules/editor/project-init-handler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akvo/rsr/spa/app/modules/editor/project-init-handler.js b/akvo/rsr/spa/app/modules/editor/project-init-handler.js index 7ef71c9e7f..b234e96a26 100644 --- a/akvo/rsr/spa/app/modules/editor/project-init-handler.js +++ b/akvo/rsr/spa/app/modules/editor/project-init-handler.js @@ -112,7 +112,7 @@ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { * Once section1 is fetched then * Update each section that has a dependency with section1 */ - if (editorRdr.section1.isFetched) { + if (editorRdr?.section1?.fields?.id) { sectionInstanceToRoot.forEach((index) => { if (!editorRdr[`section${index}`]?.isFetched) { props.fetchSectionRoot(index) From 9ce5c9204443e3bb3769328fe0367e20641d4192 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Oct 2022 14:00:14 +0200 Subject: [PATCH 08/59] refactor: move common aggregation status tests into one test #5140: Stabilize period update aggregations --- akvo/rsr/models/cron_job.py | 2 +- .../tests/usecases/jobs/test_aggregation.py | 80 +++++++------------ 2 files changed, 30 insertions(+), 52 deletions(-) diff --git a/akvo/rsr/models/cron_job.py b/akvo/rsr/models/cron_job.py index f72adf7745..9b81194ed9 100644 --- a/akvo/rsr/models/cron_job.py +++ b/akvo/rsr/models/cron_job.py @@ -43,7 +43,7 @@ def mark_failed(self): self.pid = None self.save() - def mark_max_reached(self): + def mark_maxxed(self): self.status = self.Status.MAXXED self.pid = None self.save() diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index d4b48ce00f..5f2750001f 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -3,7 +3,7 @@ from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob from akvo.rsr.tests.base import BaseTestCase -from akvo.rsr.usecases.jobs.aggregation import get_failed_jobs, get_finished_jobs, get_running_jobs, get_scheduled_jobs +from akvo.rsr.usecases.jobs import aggregation as usecases from akvo.rsr.usecases.jobs.cron import is_job_dead @@ -35,47 +35,47 @@ def setUp(self): self.job = IndicatorUpdateAggregationJob.objects.create(period=self.period, program=self.project) -class ScheduledJobTestCase(AggregationJobBaseTests): +class JobStatusTestCase(AggregationJobBaseTests): - def test_get_scheduled(self): - self.assertEqual(self.job, get_scheduled_jobs().first()) + def test_status_change(self): + """ + Ensure that setting a status and querying for it works + """ + statuses = [status.name for status in IndicatorUpdateAggregationJob.Status] - def test_not_scheduled(self): - self.job.mark_running() - self.assertIsNone(get_scheduled_jobs().first()) + for status in statuses: + with self.subTest(status): + _status = status.lower() - self.job.mark_finished() - self.assertIsNone(get_scheduled_jobs().first()) + # Mark the status + getattr(self.job, f"mark_{_status}")() - self.job.mark_failed() - self.assertIsNone(get_scheduled_jobs().first()) + # Check the retrieval + jobs = getattr(usecases, f"get_{_status}_jobs")() + self.assertEqual(self.job, jobs.first(), f"{_status} not found in its query") + # Check other it's not in the other statuses + status_set = set(statuses) + status_set.remove(status) -class RunningJobTestCase(AggregationJobBaseTests): + for other_status in status_set: + _other_status = other_status.lower() + jobs = getattr(usecases, f"get_{_other_status}_jobs")() + self.assertIsNone(jobs.first(), f"{_status} found in {_other_status} query") - def test_is_running(self): + +class RunningJobTestCase(AggregationJobBaseTests): + def setUp(self): + super().setUp() self.job.mark_running() - self.assertEqual(self.job, get_running_jobs().first()) + def test_pid_set(self): self.assertIsNotNone(self.job.pid) - def test_not_running(self): - self.job.mark_scheduled() - self.assertIsNone(get_running_jobs().first()) - - self.job.mark_finished() - self.assertIsNone(get_running_jobs().first()) - - self.job.mark_failed() - self.assertIsNone(get_running_jobs().first()) - def test_is_alive(self): - self.job.mark_running() - self.assertFalse(is_job_dead(self.job)) def test_is_dead(self): - self.job.mark_running() self.job.pid = 1_000_000 self.job.save() @@ -84,39 +84,17 @@ def test_is_dead(self): class FailedJobTestCase(AggregationJobBaseTests): - def test_is_failed(self): + def test_pid_and_attempts_attrs(self): self.job.mark_failed() - self.assertEqual(self.job, get_failed_jobs().first()) self.assertIsNone(self.job.pid) self.assertEqual(self.job.attempts, 1) - def test_not_failed(self): - self.job.mark_scheduled() - self.assertIsNone(get_failed_jobs().first()) - - self.job.mark_running() - self.assertIsNone(get_failed_jobs().first()) - - self.job.mark_finished() - self.assertIsNone(get_failed_jobs().first()) - class FinishedJobTestCase(AggregationJobBaseTests): - def test_is_finished(self): + def test_pid_and_attempts_attrs(self): self.job.mark_finished() - self.assertEqual(self.job, get_finished_jobs().first()) self.assertIsNone(self.job.pid) self.assertEqual(self.job.attempts, 1) - - def test_not_finished(self): - self.job.mark_scheduled() - self.assertIsNone(get_finished_jobs().first()) - - self.job.mark_running() - self.assertIsNone(get_finished_jobs().first()) - - self.job.mark_failed() - self.assertIsNone(get_finished_jobs().first()) From f4a4d7d3d110036a6b013af7022fe494433d005a Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Oct 2022 16:15:02 +0200 Subject: [PATCH 09/59] test: scheduling an aggregation job from an indicator period #5140: Stabilize period update aggregations --- .../tests/usecases/jobs/test_aggregation.py | 22 +++++++++++++++++++ akvo/rsr/usecases/jobs/aggregation.py | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 5f2750001f..d938fcc4d0 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -98,3 +98,25 @@ def test_pid_and_attempts_attrs(self): self.assertIsNone(self.job.pid) self.assertEqual(self.job.attempts, 1) + + +class AggregationJobScheduling(AggregationJobBaseTests): + + def test_no_existing_job(self): + """Without an existing job, a new one should be created""" + self.job.delete() + new_job = usecases.schedule_aggregation_job(self.period) + + scheduled_jobs = usecases.get_scheduled_jobs() + self.assertEqual(scheduled_jobs.count(), 1) + self.assertEqual(scheduled_jobs.first(), new_job) + + def test_existing_period_job(self): + """If a job is already scheduled, it should only be updated""" + job = usecases.schedule_aggregation_job(self.period) + + scheduled_jobs = usecases.get_scheduled_jobs() + self.assertEqual(scheduled_jobs.count(), 1) + self.assertEqual(scheduled_jobs.first(), job) + self.assertEqual(self.job, job) + self.assertNotEqual(self.job.updated_at, job.updated_at) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 3049456f6a..07b8156f1d 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -43,8 +43,11 @@ def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorUpdateAggregat Schedule a job for the period to be aggregated upwards if no job exists """ if existing_job := get_scheduled_jobs().filter(period=period).first(): + existing_job.save() return existing_job - return IndicatorUpdateAggregationJob.objects.create(period=period) + + program = period.indicator.result.project.ancestor() + return IndicatorUpdateAggregationJob.objects.create(period=period, program=program) def execute_aggregation_jobs(): From f82249c8f6b0b813fc273db46affa9d1e586ee32 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 17 Oct 2022 16:27:23 +0200 Subject: [PATCH 10/59] test: fail dead aggregation jobs Function was also renamed to be more descriptive of its actual function #5140: Stabilize period update aggregations --- .../tests/usecases/jobs/test_aggregation.py | 20 +++++++++++++++++++ akvo/rsr/usecases/jobs/aggregation.py | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index d938fcc4d0..55f4a8cda8 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -120,3 +120,23 @@ def test_existing_period_job(self): self.assertEqual(scheduled_jobs.first(), job) self.assertEqual(self.job, job) self.assertNotEqual(self.job.updated_at, job.updated_at) + + +class FailDeadJobTest(AggregationJobBaseTests): + + def test_with_dead_job(self): + # Mock a failed job + self.job.mark_running() + self.job.pid = 1_000_000 + self.job.save() + + dead_jobs = usecases.fail_dead_jobs() + + self.assertListEqual(dead_jobs, [self.job]) + + def test_with_live_job(self): + self.job.mark_running() + + dead_jobs = usecases.fail_dead_jobs() + + self.assertListEqual(dead_jobs, []) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 07b8156f1d..10eae7bb79 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -71,7 +71,7 @@ def execute_aggregation_jobs(): @atomic def handle_failed_jobs(): """Identify failed jobs, notify owners, and reschedule them""" - identify_dead_jobs() + fail_dead_jobs() failed_jobs = get_failed_jobs() for failed_job in failed_jobs: @@ -81,9 +81,9 @@ def handle_failed_jobs(): @atomic -def identify_dead_jobs() -> List[IndicatorUpdateAggregationJob]: +def fail_dead_jobs() -> List[IndicatorUpdateAggregationJob]: """ - Find jobs that are supposed to be running, but with a dead process + Find jobs that are supposed to be running but with a dead process and fail them """ dead_jobs = [] for running_job in get_running_jobs(): From 329ccf9d7423fa8aac2113ccc69aa0f232ed4f4e Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 18 Oct 2022 09:05:46 +0200 Subject: [PATCH 11/59] refactor: Rename IndicatorUpdateAggregationJob to IndicatorPeriodAggregationJob Also updated the migration #5140: Stabilize period update aggregations --- ... => 0221_indicatorperiodaggregationjob.py} | 6 ++-- akvo/rsr/models/__init__.py | 4 +-- akvo/rsr/models/aggregation_job.py | 2 +- .../tests/usecases/jobs/test_aggregation.py | 6 ++-- akvo/rsr/usecases/jobs/aggregation.py | 30 +++++++++---------- 5 files changed, 24 insertions(+), 24 deletions(-) rename akvo/rsr/migrations/{0221_indicatorupdateaggregationjob.py => 0221_indicatorperiodaggregationjob.py} (83%) diff --git a/akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py similarity index 83% rename from akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py rename to akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py index 500cd575ac..a3354bbb05 100644 --- a/akvo/rsr/migrations/0221_indicatorupdateaggregationjob.py +++ b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.10 on 2022-10-12 12:49 +# Generated by Django 3.2.10 on 2022-10-18 07:03 from django.db import migrations, models import django.db.models.deletion @@ -12,10 +12,10 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='IndicatorUpdateAggregationJob', + name='IndicatorPeriodAggregationJob', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('status', models.CharField(choices=[('SCHEDULED', 'Scheduled'), ('RUNNING', 'Running'), ('FINISHED', 'Finished'), ('FAILED', 'Failed'), ('dododod', 'Failed')], default='SCHEDULED', max_length=10)), + ('status', models.CharField(choices=[('SCHEDULED', 'Scheduled'), ('RUNNING', 'Running'), ('FINISHED', 'Finished'), ('FAILED', 'Failed'), ('MAXXED', 'Max attempts reached')], default='SCHEDULED', max_length=15)), ('attempts', models.IntegerField(default=0)), ('pid', models.PositiveIntegerField(null=True)), ('updated_at', models.DateTimeField(auto_now=True)), diff --git a/akvo/rsr/models/__init__.py b/akvo/rsr/models/__init__.py index bc4a3cf187..86c43fc905 100644 --- a/akvo/rsr/models/__init__.py +++ b/akvo/rsr/models/__init__.py @@ -18,7 +18,7 @@ update_project_budget, update_project_funding ) -from .aggregation_job import IndicatorUpdateAggregationJob +from .aggregation_job import IndicatorPeriodAggregationJob from .benchmark import Benchmark, Benchmarkname from .budget_item import BudgetItem, BudgetItemLabel, CountryBudgetItem from .country import Country, RecipientCountry @@ -171,7 +171,7 @@ 'Partnership', 'PeriodActualValue', 'PeriodDisaggregation', - 'IndicatorUpdateAggregationJob', + 'IndicatorPeriodAggregationJob', 'IndicatorPeriodDisaggregation', 'PlannedDisbursement', 'PolicyMarker', diff --git a/akvo/rsr/models/aggregation_job.py b/akvo/rsr/models/aggregation_job.py index bc7bca3e1c..a00a90c4aa 100644 --- a/akvo/rsr/models/aggregation_job.py +++ b/akvo/rsr/models/aggregation_job.py @@ -3,6 +3,6 @@ from akvo.rsr.models.cron_job import CronJobMixin -class IndicatorUpdateAggregationJob(CronJobMixin): +class IndicatorPeriodAggregationJob(CronJobMixin): period = models.ForeignKey("IndicatorPeriod", on_delete=models.CASCADE) program = models.ForeignKey("Project", on_delete=models.CASCADE) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 55f4a8cda8..127b7d515d 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -1,7 +1,7 @@ import datetime from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User -from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob +from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.usecases.jobs import aggregation as usecases from akvo.rsr.usecases.jobs.cron import is_job_dead @@ -32,7 +32,7 @@ def setUp(self): period_end=datetime.date.today() + datetime.timedelta(days=1), target_value="100" ) - self.job = IndicatorUpdateAggregationJob.objects.create(period=self.period, program=self.project) + self.job = IndicatorPeriodAggregationJob.objects.create(period=self.period, program=self.project) class JobStatusTestCase(AggregationJobBaseTests): @@ -41,7 +41,7 @@ def test_status_change(self): """ Ensure that setting a status and querying for it works """ - statuses = [status.name for status in IndicatorUpdateAggregationJob.Status] + statuses = [status.name for status in IndicatorPeriodAggregationJob.Status] for status in statuses: with self.subTest(status): diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 10eae7bb79..e713841266 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -3,42 +3,42 @@ from typing import List from akvo.rsr.models import IndicatorPeriod -from akvo.rsr.models.aggregation_job import IndicatorUpdateAggregationJob +from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob from akvo.rsr.models.cron_job import CronJobMixin from akvo.rsr.usecases.jobs.cron import is_job_dead -def get_scheduled_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: - return IndicatorUpdateAggregationJob.objects.filter( +def get_scheduled_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.filter( status=CronJobMixin.Status.SCHEDULED, ) -def get_running_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: - return IndicatorUpdateAggregationJob.objects.filter( +def get_running_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.filter( status=CronJobMixin.Status.RUNNING, ) -def get_failed_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: - return IndicatorUpdateAggregationJob.objects.filter( +def get_failed_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.filter( status=CronJobMixin.Status.FAILED, ) -def get_maxxed_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: - return IndicatorUpdateAggregationJob.objects.filter( +def get_maxxed_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.filter( status=CronJobMixin.Status.MAXXED, ) -def get_finished_jobs() -> QuerySet[IndicatorUpdateAggregationJob]: - return IndicatorUpdateAggregationJob.objects.filter( +def get_finished_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.filter( status=CronJobMixin.Status.FINISHED, ) -def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorUpdateAggregationJob: +def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorPeriodAggregationJob: """ Schedule a job for the period to be aggregated upwards if no job exists """ @@ -47,7 +47,7 @@ def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorUpdateAggregat return existing_job program = period.indicator.result.project.ancestor() - return IndicatorUpdateAggregationJob.objects.create(period=period, program=program) + return IndicatorPeriodAggregationJob.objects.create(period=period, program=program) def execute_aggregation_jobs(): @@ -81,7 +81,7 @@ def handle_failed_jobs(): @atomic -def fail_dead_jobs() -> List[IndicatorUpdateAggregationJob]: +def fail_dead_jobs() -> List[IndicatorPeriodAggregationJob]: """ Find jobs that are supposed to be running but with a dead process and fail them """ @@ -95,7 +95,7 @@ def fail_dead_jobs() -> List[IndicatorUpdateAggregationJob]: return dead_jobs -def email_failed_job_owner(failed_job: IndicatorUpdateAggregationJob, reason: str): +def email_failed_job_owner(failed_job: IndicatorPeriodAggregationJob, reason: str): raise NotImplementedError() From 7e4353fd1bf7260f913e93a04c20e06d93655d49 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Fri, 14 Oct 2022 10:40:45 +0700 Subject: [PATCH 12/59] [#5140] Disable legacy disaggregation --- akvo/rsr/models/result/disaggregation.py | 38 ++++++------ .../models/result/indicator_period_data.py | 2 +- .../rsr/tests/iati_export/test_iati_export.py | 2 + akvo/rsr/tests/rest/test_permissions.py | 4 ++ akvo/rsr/tests/rest/test_project_overview.py | 6 +- .../results_framework/test_aggregation.py | 9 +++ .../test_disaggregation_aggregation.py | 13 ++++ .../test_disaggregation_contribution.py | 6 ++ .../test_results_framework.py | 5 ++ .../test_period_update_aggregation.py | 60 +++++++++++++++++++ .../rsr/usecases/period_update_aggregation.py | 5 ++ 11 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 akvo/rsr/tests/usecases/test_period_update_aggregation.py create mode 100644 akvo/rsr/usecases/period_update_aggregation.py diff --git a/akvo/rsr/models/result/disaggregation.py b/akvo/rsr/models/result/disaggregation.py index 9a6cb427bb..6b2036f846 100644 --- a/akvo/rsr/models/result/disaggregation.py +++ b/akvo/rsr/models/result/disaggregation.py @@ -70,25 +70,25 @@ def update_incomplete_data(self): self.siblings().update(incomplete_data=incomplete_data) -@receiver([signals.post_save, signals.post_delete], sender=Disaggregation) -def aggregate_period_disaggregation_up_to_parent_hierarchy(sender, **kwargs): - - # Disable signal handler when loading fixtures - if kwargs.get('raw', False): - return - - from .disaggregation_aggregation import DisaggregationAggregation - from .indicator_period_disaggregation import IndicatorPeriodDisaggregation - - disaggregation = kwargs['instance'] - disaggregation_aggregation = DisaggregationAggregation( - Disaggregation.objects, - IndicatorPeriodDisaggregation.objects - ) - disaggregation_aggregation.aggregate( - disaggregation.update.period, - disaggregation.dimension_value - ) +# @receiver([signals.post_save, signals.post_delete], sender=Disaggregation) +# def aggregate_period_disaggregation_up_to_parent_hierarchy(sender, **kwargs): +# +# # Disable signal handler when loading fixtures +# if kwargs.get('raw', False): +# return +# +# from .disaggregation_aggregation import DisaggregationAggregation +# from .indicator_period_disaggregation import IndicatorPeriodDisaggregation +# +# disaggregation = kwargs['instance'] +# disaggregation_aggregation = DisaggregationAggregation( +# Disaggregation.objects, +# IndicatorPeriodDisaggregation.objects +# ) +# disaggregation_aggregation.aggregate( +# disaggregation.update.period, +# disaggregation.dimension_value +# ) @receiver(signals.post_save, sender=Disaggregation) diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index 77c08d6be2..9b4e922f17 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -101,7 +101,7 @@ def save(self, recalculate=True, *args, **kwargs): # In case the status is approved, recalculate the period if recalculate and self.status == self.STATUS_APPROVED_CODE: # FIXME: Should we call this even when status is not approved? - self.period.recalculate_period() + # self.period.recalculate_period() self.period.update_actual_comment() # Update score even when the update is not approved, yet. It handles the # case where an approved update is returned for revision, etc. diff --git a/akvo/rsr/tests/iati_export/test_iati_export.py b/akvo/rsr/tests/iati_export/test_iati_export.py index b9ccdc636c..1bdbd19eff 100644 --- a/akvo/rsr/tests/iati_export/test_iati_export.py +++ b/akvo/rsr/tests/iati_export/test_iati_export.py @@ -10,6 +10,7 @@ import datetime import os import shutil +import unittest from lxml import etree from django.core.files.uploadedfile import SimpleUploadedFile @@ -399,6 +400,7 @@ def setUp(self): self.project = project self.related_project = related_project + @unittest.skip('aggregation behaviour refactoring') def test_complete_project_export(self): """ Test the export of a fully filled project. diff --git a/akvo/rsr/tests/rest/test_permissions.py b/akvo/rsr/tests/rest/test_permissions.py index a1e80db2e7..58e7af81b7 100644 --- a/akvo/rsr/tests/rest/test_permissions.py +++ b/akvo/rsr/tests/rest/test_permissions.py @@ -9,6 +9,7 @@ import collections import re +import unittest from django.conf import settings from django.contrib.auth import get_user_model @@ -612,6 +613,7 @@ def set_publishing_status(project, status): publishing_status.status = status publishing_status.save() + @unittest.skip('aggregation behaviour refactoring') def test_admin(self): for user in User.objects.filter(is_admin=True): for queryset, project_relation, count in self.iter_queryset('admin'): @@ -620,6 +622,7 @@ def test_admin(self): ) self.assertPermissions(user, count, filtered_queryset) + @unittest.skip('aggregation behaviour refactoring') def test_anonymous(self): user = AnonymousUser() for queryset, project_relation, count in self.iter_queryset('anonymous'): @@ -628,6 +631,7 @@ def test_anonymous(self): ) self.assertPermissions(user, count, filtered_queryset) + @unittest.skip('aggregation behaviour refactoring') def test_logged_in_user(self): m_e = Group.objects.get(name='M&E Managers') p_e = Group.objects.get(name='Project Editors') diff --git a/akvo/rsr/tests/rest/test_project_overview.py b/akvo/rsr/tests/rest/test_project_overview.py index df28f62e4a..cf2131bf81 100644 --- a/akvo/rsr/tests/rest/test_project_overview.py +++ b/akvo/rsr/tests/rest/test_project_overview.py @@ -4,7 +4,7 @@ # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. - +import unittest from unittest.mock import patch from akvo.rsr.models import Result, Indicator, IndicatorPeriod, IndicatorPeriodData @@ -13,6 +13,7 @@ class QuantitativeUnitAggregationTestCase(BaseTestCase): + @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -31,6 +32,7 @@ def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self) final_value = response.data['indicators'][0]['periods'][0]['actual_value'] self.assertEqual(final_value, 1 + 2) + @unittest.skip('aggregation behaviour refactoring') def test_updates_from_every_level_of_hierarchy_are_calculated_for_final_value(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -146,6 +148,7 @@ def test_handle_update_with_null_value(self): class QuantitativePercentageAggregationTestCase(BaseTestCase): + @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -309,6 +312,7 @@ def test_handle_update_with_null_value(self): class QualitativeScoresAggregationTestCase(BaseTestCase): + @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ diff --git a/akvo/rsr/tests/results_framework/test_aggregation.py b/akvo/rsr/tests/results_framework/test_aggregation.py index d7f5744bd5..321cdfcb90 100644 --- a/akvo/rsr/tests/results_framework/test_aggregation.py +++ b/akvo/rsr/tests/results_framework/test_aggregation.py @@ -147,6 +147,7 @@ def test_should_update_actual_value_with_update_data(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(str(actual_value + increment), period.actual_value) + @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_update_numeric_data(self): # Given increment = 2 @@ -159,6 +160,7 @@ def test_should_aggregate_update_numeric_data(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(increment * 2), Decimal(period.actual_value)) + @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_update_str_negative_data(self): # Given original = 5 @@ -188,6 +190,7 @@ def test_should_replace_with_non_numeric_update_data(self): # Single child tests + @unittest.skip('aggregation behaviour refactoring') def test_should_copy_child_period_value(self): # Given value = 5 @@ -207,6 +210,7 @@ def test_should_copy_child_period_value(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(child_value), Decimal(period.actual_value)) + @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_child_period_value(self): # Given self.parent_project.aggregate_children = False @@ -230,6 +234,7 @@ def test_should_not_aggregate_child_period_value(self): # Multiple children tests + @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_child_indicators_values(self): # Given child_project_2 = self.create_child_project('Child project 2') @@ -247,6 +252,7 @@ def test_should_aggregate_child_indicators_values(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(value * 2), Decimal(period.actual_value)) + @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_excluded_child_period_values(self): # Given child_project_2 = self.create_child_project('Child project 2') @@ -329,6 +335,7 @@ def test_should_replace_with_non_numeric_update_data(self): # Single child tests + @unittest.skip('aggregation behaviour refactoring') def test_should_copy_child_period_value(self): # Given numerator = 5 @@ -387,6 +394,7 @@ def test_should_not_aggregate_child_period_value(self): # Multiple children tests + @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_child_indicators_values(self): # Given numerator = 5 @@ -421,6 +429,7 @@ def test_should_aggregate_child_indicators_values(self): self.assertDecimalEqual(child_numerator * 2, period.numerator) self.assertDecimalEqual(child_denominator * 2, period.denominator) + @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_excluded_child_period_values(self): # Given child_project_2 = self.create_child_project('Child project 2') diff --git a/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py b/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py index 3563afa5d3..6c327a3b58 100644 --- a/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py +++ b/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py @@ -1,5 +1,6 @@ import datetime +import unittest from akvo.rsr.models import ( Result, Indicator, IndicatorPeriod, IndicatorDimensionName, @@ -28,6 +29,7 @@ def setUp(self): IndicatorDimensionValue.objects.create(name=category, value=self.type2) self.indicator.dimension_names.add(category) + @unittest.skip('aggregation behaviour refactoring') def test_aggregate_to_period(self): # Given disaggregations = util.get_disaggregations(self.project) @@ -65,6 +67,7 @@ def test_aggregate_to_period(self): 30 + 15 ) + @unittest.skip('aggregation behaviour refactoring') def test_aggregate_percentages_to_period(self): # Given disaggregations = util.get_disaggregations(self.project) @@ -104,6 +107,7 @@ def test_aggregate_percentages_to_period(self): self.assertEqual(period_disaggregation2.numerator, 30 + 15) self.assertEqual(period_disaggregation2.denominator, 50 + 50) + @unittest.skip('aggregation behaviour refactoring') def test_aggregate_child_period_to_parent(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -138,6 +142,7 @@ def test_aggregate_child_period_to_parent(self): 30 + 15 ) + @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_unapproved_updates(self): # Given child = self.create_contributor("Child", self.project) @@ -178,6 +183,7 @@ def test_should_not_aggregate_unapproved_updates(self): None ) + @unittest.skip('aggregation behaviour refactoring') def test_project_with_aggregate_to_parent_off(self): # Given child = self.create_contributor("Child", self.project) @@ -212,6 +218,7 @@ def test_project_with_aggregate_to_parent_off(self): 30 ) + @unittest.skip('aggregation behaviour refactoring') def test_project_with_aggregate_children_off(self): # Given self.project.aggregate_children = False @@ -246,6 +253,7 @@ def test_project_with_aggregate_children_off(self): 30 ) + @unittest.skip('aggregation behaviour refactoring') def test_aggregate_multi_level_hierarchy(self): # Given child = self.create_contributor("Child", self.project) @@ -298,6 +306,7 @@ def test_aggregate_multi_level_hierarchy(self): 30 + 15 ) + @unittest.skip('aggregation behaviour refactoring') def test_aggregate_multiple_periods(self): # Given period2 = IndicatorPeriod.objects.create( @@ -362,6 +371,7 @@ def test_aggregate_multiple_periods(self): 15 ) + @unittest.skip('aggregation behaviour refactoring') def test_sum_local_and_aggregated_updates(self): # Given child = self.create_contributor("Child", self.project) @@ -417,6 +427,7 @@ def test_sum_local_and_aggregated_updates(self): 30 + 15 ) + @unittest.skip('aggregation behaviour refactoring') def test_amend_on_the_lowest_level_case1(self): # Given child = self.create_contributor("Child", self.project) @@ -471,6 +482,7 @@ def test_amend_on_the_lowest_level_case1(self): 30 + 15 + 30 ) + @unittest.skip('aggregation behaviour refactoring') def test_amend_on_the_lowest_level_case2(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -538,6 +550,7 @@ def test_amend_on_the_lowest_level_case2(self): 30 ) + @unittest.skip('aggregation behaviour refactoring') def test_delete_on_the_lowest_level(self): # Given child1 = self.create_contributor("Child 1", self.project) diff --git a/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py b/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py index ce203186f4..42038a852a 100644 --- a/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py +++ b/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py @@ -8,6 +8,7 @@ """ import datetime +import unittest from akvo.rsr.models import ( Result, Indicator, IndicatorPeriod, IndicatorDimensionName, IndicatorDimensionValue) @@ -33,6 +34,7 @@ def setUp(self): IndicatorDimensionValue.objects.create(name=category, value=self.type) indicator.dimension_names.add(category) + @unittest.skip('aggregation behaviour refactoring') def test_no_contribution(self): # Given type = util.get_disaggregations(self.project).filter(value=self.type).first() @@ -50,6 +52,7 @@ def test_no_contribution(self): util.get_disaggregation_contributors(self.period, self.type).count(), 0) + @unittest.skip('aggregation behaviour refactoring') def test_disaggregation_contribution_from_child_to_parent(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -79,6 +82,7 @@ def test_disaggregation_contribution_from_child_to_parent(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 15) + @unittest.skip('aggregation behaviour refactoring') def test_multi_level_disaggregation_contribution(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -114,6 +118,7 @@ def test_multi_level_disaggregation_contribution(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 15) + @unittest.skip('aggregation behaviour refactoring') def test_amend_disaggregation_contributions(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -151,6 +156,7 @@ def test_amend_disaggregation_contributions(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 30) + @unittest.skip('aggregation behaviour refactoring') def test_delete_period_update_contributions(self): # Given child1 = self.create_contributor("Child 1", self.project) diff --git a/akvo/rsr/tests/results_framework/test_results_framework.py b/akvo/rsr/tests/results_framework/test_results_framework.py index 2aef4f152c..bdcb00dac5 100644 --- a/akvo/rsr/tests/results_framework/test_results_framework.py +++ b/akvo/rsr/tests/results_framework/test_results_framework.py @@ -521,6 +521,7 @@ def test_child_dimension_value_delete_prevented(self): with self.assertRaises(PermissionDenied): child_dimension_name.dimension_values.last().delete() + @unittest.skip('aggregation behaviour refactoring') def test_update(self): """ Test if placing updates will update the actual value of the period. @@ -547,6 +548,7 @@ def test_update(self): indicator_update_2.save() self.assertEqual(self.period.actual_value, "15.00") + @unittest.skip('aggregation behaviour refactoring') def test_edit_and_delete_updates(self): """ Test if editing or deleting updates will update the actual value of the period. @@ -569,6 +571,7 @@ def test_edit_and_delete_updates(self): indicator_update.delete() self.assertEqual(self.period.actual_value, "0") + @unittest.skip('aggregation behaviour refactoring') def test_update_on_child(self): """ Test if placing an update on the child project will update the actual value of the period, @@ -603,6 +606,7 @@ def test_update_on_child(self): indicator__result__project=self.parent_project).first() self.assertEqual(parent_period.actual_value, "25.00") + @unittest.skip('aggregation behaviour refactoring') def test_update_without_aggregations(self): """ Test if placing an update on the child project without an aggregation will not update the @@ -641,6 +645,7 @@ def test_update_without_aggregations(self): indicator__result__project=self.parent_project).first() self.assertEqual(parent_period.actual_value, "10.00") + @unittest.skip('aggregation behaviour refactoring') def test_updates_with_percentages(self): """ Test if placing an update on two child projects will give the average of the two in the diff --git a/akvo/rsr/tests/usecases/test_period_update_aggregation.py b/akvo/rsr/tests/usecases/test_period_update_aggregation.py new file mode 100644 index 0000000000..b14ae732ac --- /dev/null +++ b/akvo/rsr/tests/usecases/test_period_update_aggregation.py @@ -0,0 +1,60 @@ +from datetime import date +from akvo.rsr.usecases.period_update_aggregation import aggregate +from akvo.rsr.tests.base import BaseTestCase +from akvo.rsr.tests.utils import ProjectFixtureBuilder + + +class PeriodUpdateAggregationTestCase(BaseTestCase): + CONTRIBUTOR_LEVEL_1 = 'L1 contributor' + CONTRIBUTOR_LEVEL_2 = 'L2 contributor' + DISAGGREGATION_CATEGORY = 'Gender' + DISAGGREGATION_TYPE_1 = 'Male' + DISAGGREGATION_TYPE_2 = 'Female' + PERIOD_START = date(2020, 1, 1) + + def setUp(self): + super().setUp() + user = self.create_user('test@akvo.org', 'password', is_admin=True) + self.lead_project = ProjectFixtureBuilder().with_contributors([{ + 'title': self.CONTRIBUTOR_LEVEL_1, + 'contributors': [ + {'title': self.CONTRIBUTOR_LEVEL_2} + ] + }]).with_disaggregations({ + self.DISAGGREGATION_CATEGORY: [ + self.DISAGGREGATION_TYPE_1, + self.DISAGGREGATION_TYPE_2, + ] + }).with_results([{ + 'title': 'Result #1', + 'indicators': [{ + 'title': 'Indicator #1', + 'periods': [ + {'period_start': self.PERIOD_START, 'period_end': date(2020, 12, 31)}, + ] + }] + }]).build() + + self.l1_contributor = self.lead_project.get_contributor(title=self.CONTRIBUTOR_LEVEL_1) + self.l2_contributor = self.l1_contributor.get_contributor(title=self.CONTRIBUTOR_LEVEL_2) + self.l2_period = self.l2_contributor.get_period(period_start=self.PERIOD_START) + + self.l2_period.add_update(user=user, value=2, disaggregations={ + self.DISAGGREGATION_CATEGORY: { + self.DISAGGREGATION_TYPE_1: {'value': 1}, + self.DISAGGREGATION_TYPE_2: {'value': 1}, + } + }) + + def test_not_automatically_aggregated(self): + ''' Ensure the legacy behavior that performs aggregation when period updates are saved is not running ''' + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) + l2_period = self.l2_contributor.periods.get(period_start=self.PERIOD_START) + + self.assertEqual('', l2_period.actual_value) + self.assertEqual(0, l2_period.disaggregations.count()) + self.assertEqual('', l1_period.actual_value) + self.assertEqual(0, l1_period.disaggregations.count()) + self.assertEqual('', lead_period.actual_value) + self.assertEqual(0, lead_period.disaggregations.count()) diff --git a/akvo/rsr/usecases/period_update_aggregation.py b/akvo/rsr/usecases/period_update_aggregation.py new file mode 100644 index 0000000000..156e340601 --- /dev/null +++ b/akvo/rsr/usecases/period_update_aggregation.py @@ -0,0 +1,5 @@ +from akvo.rsr.models import IndicatorPeriodData + + +def aggregate(period: IndicatorPeriodData): + pass From c0134c3f803483c34339d17ec0304fd193a1c65d Mon Sep 17 00:00:00 2001 From: zuhdil Date: Mon, 17 Oct 2022 15:28:12 +0700 Subject: [PATCH 13/59] [#5140] Implement period update aggregation function --- .../result/disaggregation_aggregation.py | 29 +--- akvo/rsr/models/result/indicator_period.py | 20 +-- .../models/result/indicator_period_data.py | 2 +- .../test_period_update_aggregation.py | 153 +++++++++++++++++- akvo/rsr/tests/utils.py | 2 + .../rsr/usecases/period_update_aggregation.py | 69 +++++++- 6 files changed, 229 insertions(+), 46 deletions(-) diff --git a/akvo/rsr/models/result/disaggregation_aggregation.py b/akvo/rsr/models/result/disaggregation_aggregation.py index 6f42cfc572..1fc94ae3fa 100644 --- a/akvo/rsr/models/result/disaggregation_aggregation.py +++ b/akvo/rsr/models/result/disaggregation_aggregation.py @@ -4,8 +4,8 @@ # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. -from decimal import Decimal, InvalidOperation from django.db.models import Sum +from akvo.utils import ensure_decimal from .indicator_period_data import IndicatorPeriodData @@ -18,7 +18,6 @@ def __init__(self, disaggregations, period_disaggregations): def aggregate(self, period, dimension_value): local = self._get_local_values(period, dimension_value) contributed = self._get_contributed_values(period, dimension_value) - period_disaggregation, _ = self.period_disaggregations.get_or_create( period=period, dimension_value=dimension_value @@ -27,7 +26,6 @@ def aggregate(self, period, dimension_value): period_disaggregation.numerator = _sum_attr(local, contributed, 'numerator') period_disaggregation.denominator = _sum_attr(local, contributed, 'denominator') period_disaggregation.save() - if period.parent_period is not None \ and period.parent_period.indicator.result.project.aggregate_children \ and period.indicator.result.project.aggregate_to_parent \ @@ -46,20 +44,10 @@ def _get_local_values(self, period, dimension_value): ) def _get_contributed_values(self, period, dimension_value): - child_dimension_values_pks = [ - child_dimension_value.id - for child_dimension_value - in dimension_value.child_dimension_values.all() - ] - child_periods_pks = [ - child_period.id - for child_period - in period.child_periods.all() - ] return self.period_disaggregations\ .filter( - dimension_value__in=child_dimension_values_pks, - period__in=child_periods_pks + dimension_value__in=dimension_value.child_dimension_values.all(), + period__in=period.child_periods.all(), ).aggregate( value=Sum('value'), numerator=Sum('numerator'), @@ -70,15 +58,6 @@ def _get_contributed_values(self, period, dimension_value): def _sum_attr(a, b, attr): left = a[attr] right = b[attr] - if left is None and right is None: return None - - return _d(left) + _d(right) - - -def _d(value): - try: - return Decimal(value) - except (InvalidOperation, TypeError): - return Decimal(0) + return ensure_decimal(left) + ensure_decimal(right) diff --git a/akvo/rsr/models/result/indicator_period.py b/akvo/rsr/models/result/indicator_period.py index 4e08a60077..36f30d4241 100644 --- a/akvo/rsr/models/result/indicator_period.py +++ b/akvo/rsr/models/result/indicator_period.py @@ -96,7 +96,7 @@ def __str__(self): return period_unicode def save(self, *args, **kwargs): - actual_value_changed = False + # actual_value_changed = False new_period = not self.pk if ( @@ -107,11 +107,11 @@ def save(self, *args, **kwargs): percentage = calculate_percentage(self.numerator, self.denominator) self.actual_value = str(percentage) - if not new_period: - # Check if the actual value has changed - orig_period = IndicatorPeriod.objects.get(pk=self.pk) - if orig_period.actual_value != self.actual_value: - actual_value_changed = True + # if not new_period: + # # Check if the actual value has changed + # orig_period = IndicatorPeriod.objects.get(pk=self.pk) + # if orig_period.actual_value != self.actual_value: + # actual_value_changed = True super(IndicatorPeriod, self).save(*args, **kwargs) @@ -128,10 +128,10 @@ def save(self, *args, **kwargs): # If the actual value has changed, the period has a parent period and aggregations are on, # then the the parent should be updated as well - if actual_value_changed and self.is_child_period() and \ - self.parent_period.indicator.result.project.aggregate_children and \ - self.indicator.result.project.aggregate_to_parent: - self.parent_period.recalculate_period() + # if actual_value_changed and self.is_child_period() and \ + # self.parent_period.indicator.result.project.aggregate_children and \ + # self.indicator.result.project.aggregate_to_parent: + # self.parent_period.recalculate_period() def clean(self): validation_errors = {} diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index 9b4e922f17..8a6542003b 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -114,7 +114,7 @@ def delete(self, *args, **kwargs): # In case the status was approved, recalculate the period if old_status == self.STATUS_APPROVED_CODE: - self.period.recalculate_period() + # self.period.recalculate_period() self.period.update_actual_comment() self.period.update_score() diff --git a/akvo/rsr/tests/usecases/test_period_update_aggregation.py b/akvo/rsr/tests/usecases/test_period_update_aggregation.py index b14ae732ac..eb9f1d1608 100644 --- a/akvo/rsr/tests/usecases/test_period_update_aggregation.py +++ b/akvo/rsr/tests/usecases/test_period_update_aggregation.py @@ -1,4 +1,7 @@ from datetime import date +from unittest.mock import patch +from akvo.utils import maybe_decimal +from akvo.rsr.models import IndicatorPeriod from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.tests.utils import ProjectFixtureBuilder @@ -6,7 +9,8 @@ class PeriodUpdateAggregationTestCase(BaseTestCase): CONTRIBUTOR_LEVEL_1 = 'L1 contributor' - CONTRIBUTOR_LEVEL_2 = 'L2 contributor' + CONTRIBUTOR_LEVEL_2_1 = 'L2 contributor #1' + CONTRIBUTOR_LEVEL_2_2 = 'L2 contributor #2' DISAGGREGATION_CATEGORY = 'Gender' DISAGGREGATION_TYPE_1 = 'Male' DISAGGREGATION_TYPE_2 = 'Female' @@ -14,11 +18,12 @@ class PeriodUpdateAggregationTestCase(BaseTestCase): def setUp(self): super().setUp() - user = self.create_user('test@akvo.org', 'password', is_admin=True) + self.user = self.create_user('test@akvo.org', 'password', is_admin=True) self.lead_project = ProjectFixtureBuilder().with_contributors([{ 'title': self.CONTRIBUTOR_LEVEL_1, 'contributors': [ - {'title': self.CONTRIBUTOR_LEVEL_2} + {'title': self.CONTRIBUTOR_LEVEL_2_1}, + {'title': self.CONTRIBUTOR_LEVEL_2_2}, ] }]).with_disaggregations({ self.DISAGGREGATION_CATEGORY: [ @@ -36,25 +41,159 @@ def setUp(self): }]).build() self.l1_contributor = self.lead_project.get_contributor(title=self.CONTRIBUTOR_LEVEL_1) - self.l2_contributor = self.l1_contributor.get_contributor(title=self.CONTRIBUTOR_LEVEL_2) - self.l2_period = self.l2_contributor.get_period(period_start=self.PERIOD_START) + self.l2_1_contributor = self.l1_contributor.get_contributor(title=self.CONTRIBUTOR_LEVEL_2_1) + self.l2_2_contributor = self.l1_contributor.get_contributor(title=self.CONTRIBUTOR_LEVEL_2_2) + self.l2_1_period = self.l2_1_contributor.get_period(period_start=self.PERIOD_START) + self.l2_2_period = self.l2_2_contributor.get_period(period_start=self.PERIOD_START) - self.l2_period.add_update(user=user, value=2, disaggregations={ + self.l2_1_period.add_update(user=self.user, value=2, disaggregations={ self.DISAGGREGATION_CATEGORY: { self.DISAGGREGATION_TYPE_1: {'value': 1}, self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + self.l2_2_period.add_update(user=self.user, value=3, disaggregations={ + self.DISAGGREGATION_CATEGORY: { + self.DISAGGREGATION_TYPE_1: {'value': 2}, + self.DISAGGREGATION_TYPE_2: {'value': 1}, + } + }) + + def get_disaggregation_value(self, period, disaggregation_type): + return period.disaggregations.get(dimension_value__value=disaggregation_type).value def test_not_automatically_aggregated(self): ''' Ensure the legacy behavior that performs aggregation when period updates are saved is not running ''' + l2_1_period = self.l2_1_contributor.periods.get(period_start=self.PERIOD_START) + self.assertEqual('', l2_1_period.actual_value) + self.assertEqual(0, l2_1_period.disaggregations.count()) + + l2_2_period = self.l2_2_contributor.periods.get(period_start=self.PERIOD_START) + self.assertEqual('', l2_2_period.actual_value) + self.assertEqual(0, l2_2_period.disaggregations.count()) + + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) + self.assertEqual('', l1_period.actual_value) + self.assertEqual(0, l1_period.disaggregations.count()) + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + self.assertEqual('', lead_period.actual_value) + self.assertEqual(0, lead_period.disaggregations.count()) + + def test_aggregation(self): + ''' Aggregated update from all Level 2 contributors ''' + l2_1_period = self.l2_1_contributor.periods.get(period_start=self.PERIOD_START) + l2_2_period = self.l2_2_contributor.periods.get(period_start=self.PERIOD_START) + aggregate(l2_1_period) + aggregate(l2_2_period) + + l2_1_period.refresh_from_db() + self.assertEqual(2, maybe_decimal(l2_1_period.actual_value)) + self.assertEqual(1, self.get_disaggregation_value(l2_1_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(l2_1_period, self.DISAGGREGATION_TYPE_2)) + + l2_2_period.refresh_from_db() + self.assertEqual(3, maybe_decimal(l2_2_period.actual_value)) + self.assertEqual(2, self.get_disaggregation_value(l2_2_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(l2_2_period, self.DISAGGREGATION_TYPE_2)) + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) - l2_period = self.l2_contributor.periods.get(period_start=self.PERIOD_START) + self.assertEqual(5, maybe_decimal(l1_period.actual_value)) + self.assertEqual(3, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(2, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_2)) + + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + self.assertEqual(5, maybe_decimal(lead_period.actual_value)) + self.assertEqual(3, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(2, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_2)) + + def test_rollback(self): + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + l2_period = self.l2_1_contributor.periods.get(period_start=self.PERIOD_START) + orig = IndicatorPeriod.save + def mocked_indicator_period_save(self, *args, **kwargs): + if self == lead_period: + raise Exception('Rollback transaction!') + return orig(self, *args, **kwargs) + + with patch('akvo.rsr.models.result.indicator_period.IndicatorPeriod.save', new=mocked_indicator_period_save): + try: + aggregate(l2_period) + except Exception: + pass + + l2_period.refresh_from_db() self.assertEqual('', l2_period.actual_value) self.assertEqual(0, l2_period.disaggregations.count()) + + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) self.assertEqual('', l1_period.actual_value) self.assertEqual(0, l1_period.disaggregations.count()) + + lead_period.refresh_from_db() self.assertEqual('', lead_period.actual_value) self.assertEqual(0, lead_period.disaggregations.count()) + + def test_level_1_has_update(self): + ''' Updates on level 1 project should be aggregated ''' + self.l1_contributor.get_period(period_start=self.PERIOD_START).add_update( + user=self.user, + value=3, + disaggregations={ + self.DISAGGREGATION_CATEGORY: { + self.DISAGGREGATION_TYPE_1: {'value': 2}, + self.DISAGGREGATION_TYPE_2: {'value': 1}, + } + } + ) + l2_period = self.l2_1_contributor.periods.get(period_start=self.PERIOD_START) + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) + aggregate(l2_period) + aggregate(l1_period) + + l2_period.refresh_from_db() + self.assertEqual(2, maybe_decimal(l2_period.actual_value)) + self.assertEqual(1, self.get_disaggregation_value(l2_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(l2_period, self.DISAGGREGATION_TYPE_2)) + + l1_period.refresh_from_db() + self.assertEqual(5, maybe_decimal(l1_period.actual_value)) + self.assertEqual(3, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(2, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_2)) + + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + self.assertEqual(5, maybe_decimal(lead_period.actual_value)) + self.assertEqual(3, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(2, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_2)) + + def test_delete_update(self): + ''' Simulate indicator update deletion ''' + l2_1_period = self.l2_1_contributor.periods.get(period_start=self.PERIOD_START) + l2_2_period = self.l2_2_contributor.periods.get(period_start=self.PERIOD_START) + aggregate(l2_1_period) + aggregate(l2_2_period) + + l2_2_update = l2_2_period.data.first() + l2_2_update.delete() + aggregate(l2_2_period) + + l2_1_period.refresh_from_db() + self.assertEqual(2, maybe_decimal(l2_1_period.actual_value)) + self.assertEqual(1, self.get_disaggregation_value(l2_1_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(l2_1_period, self.DISAGGREGATION_TYPE_2)) + + l2_2_period.refresh_from_db() + self.assertEqual('', l2_2_period.actual_value) + self.assertEqual(None, self.get_disaggregation_value(l2_2_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(None, self.get_disaggregation_value(l2_2_period, self.DISAGGREGATION_TYPE_2)) + + l1_period = self.l1_contributor.periods.get(period_start=self.PERIOD_START) + self.assertEqual(2, maybe_decimal(l1_period.actual_value)) + self.assertEqual(1, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(l1_period, self.DISAGGREGATION_TYPE_2)) + + lead_period = self.lead_project.periods.get(period_start=self.PERIOD_START) + self.assertEqual(2, maybe_decimal(lead_period.actual_value)) + self.assertEqual(1, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_1)) + self.assertEqual(1, self.get_disaggregation_value(lead_period, self.DISAGGREGATION_TYPE_2)) diff --git a/akvo/rsr/tests/utils.py b/akvo/rsr/tests/utils.py index 115fd1a29e..4dd7b9a278 100644 --- a/akvo/rsr/tests/utils.py +++ b/akvo/rsr/tests/utils.py @@ -93,6 +93,8 @@ def _build_indicator(self, project, result, params): enumerators = kwargs.pop('enumerators', []) kwargs['result'] = result indicator = Indicator.objects.create(**kwargs) + for dimension_name in project.dimension_names.all(): + indicator.dimension_names.add(dimension_name) for params in periods: self._build_period(project, indicator, params) for enumerator in enumerators: diff --git a/akvo/rsr/usecases/period_update_aggregation.py b/akvo/rsr/usecases/period_update_aggregation.py index 156e340601..3477fb0645 100644 --- a/akvo/rsr/usecases/period_update_aggregation.py +++ b/akvo/rsr/usecases/period_update_aggregation.py @@ -1,5 +1,68 @@ -from akvo.rsr.models import IndicatorPeriodData +from decimal import Decimal +from typing import Tuple, Optional +from django.db import transaction +from django.db.models import QuerySet, Q, Sum +from akvo.utils import ensure_decimal +from akvo.rsr.models import IndicatorPeriod, IndicatorPeriodData, Disaggregation, IndicatorDimensionValue +from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE, calculate_percentage +from akvo.rsr.models.result.indicator_period_disaggregation import IndicatorPeriodDisaggregation +from akvo.rsr.models.result.disaggregation_aggregation import DisaggregationAggregation +disaggregation_aggregation = DisaggregationAggregation(Disaggregation.objects, IndicatorPeriodDisaggregation.objects) -def aggregate(period: IndicatorPeriodData): - pass + +@transaction.atomic +def aggregate(period: IndicatorPeriod): + _aggregate_period_value(period) + _aggregate_disaggregation(period) + + +def _aggregate_period_value(period: IndicatorPeriod): + value, numerator, denominator = sum_updates(period) + if period.indicator.measure == PERCENTAGE_MEASURE: + contrib_numerator, contrib_denominator = sum_contributed_percentage_value(period) + numerator = ensure_decimal(numerator) + ensure_decimal(contrib_numerator) + denominator = ensure_decimal(denominator) + ensure_decimal(contrib_denominator) + value = calculate_percentage(numerator, denominator) + else: + value = ensure_decimal(value) + sum_contributed_unit_value(period) + period.actual_value = str(value) if value else '' + if period.indicator.measure == PERCENTAGE_MEASURE: + period.numerator = numerator + period.denominator = denominator + period.save() + if period.parent_period \ + and period.indicator.result.project.aggregate_to_parent \ + and period.parent_period.indicator.result.project.aggregate_children: + _aggregate_period_value(period.parent_period) + + +def _aggregate_disaggregation(period: IndicatorPeriod): + disaggregations = Disaggregation.objects.filter(update__period=period, update__status=IndicatorPeriodData.STATUS_APPROVED_CODE) + dimension_values = ( + IndicatorDimensionValue.objects.filter(name__in=period.indicator.dimension_names.all()) + | IndicatorDimensionValue.objects.filter(disaggregations__in=disaggregations) + ).distinct() + for dimension_value in dimension_values: + disaggregation_aggregation.aggregate(period, dimension_value) + + +def sum_updates(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]: + result = period.approved_updates.aggregate(value=Sum('value'), numerator=Sum('numerator'), denominator=Sum('denominator')) + return (result[k] for k in ('value', 'numerator', 'denominator')) + + +def sum_contributed_unit_value(period: IndicatorPeriod) -> Decimal: + value = Decimal(0) + for contributor in get_contributing_child_periods(period): + value += ensure_decimal(contributor.actual_value) + return value + + +def sum_contributed_percentage_value(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal]]: + result = get_contributing_child_periods(period).aggregate(numerator=Sum('numerator'), denominator=Sum('denominator')) + return (result[k] for k in ('numerator', 'denominator')) + + +def get_contributing_child_periods(period: IndicatorPeriod) -> QuerySet: + return period.child_periods.exclude(Q(actual_value__isnull=True) | Q(actual_value__exact='')) From 868966843e5a9259fb5798df38347bb86ccf16b6 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Tue, 18 Oct 2022 16:31:40 +0700 Subject: [PATCH 14/59] [#5140] Re-enable skipped tests --- .../rsr/tests/iati_export/test_iati_export.py | 4 +- akvo/rsr/tests/rest/test_permissions.py | 8 ++-- akvo/rsr/tests/rest/test_project_overview.py | 7 +-- .../results_framework/test_aggregation.py | 25 +++++++---- .../test_disaggregation_aggregation.py | 43 +++++++++++++------ .../test_disaggregation_contribution.py | 19 +++++--- .../test_results_framework.py | 28 +++++++++--- 7 files changed, 87 insertions(+), 47 deletions(-) diff --git a/akvo/rsr/tests/iati_export/test_iati_export.py b/akvo/rsr/tests/iati_export/test_iati_export.py index 1bdbd19eff..14c22202c2 100644 --- a/akvo/rsr/tests/iati_export/test_iati_export.py +++ b/akvo/rsr/tests/iati_export/test_iati_export.py @@ -10,7 +10,6 @@ import datetime import os import shutil -import unittest from lxml import etree from django.core.files.uploadedfile import SimpleUploadedFile @@ -30,6 +29,7 @@ ProjectEditorValidationSet, IndicatorDimensionName, IndicatorDimensionValue, Disaggregation) from akvo.rsr.models.result.utils import QUALITATIVE +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.tests.iati_export import AkvoXmlMixin @@ -400,7 +400,6 @@ def setUp(self): self.project = project self.related_project = related_project - @unittest.skip('aggregation behaviour refactoring') def test_complete_project_export(self): """ Test the export of a fully filled project. @@ -494,6 +493,7 @@ def test_complete_project_export(self): update=update, value=5, ) + aggregate(period) q_period = IndicatorPeriod.objects.create( indicator=q_indicator, period_start=datetime.date.today(), diff --git a/akvo/rsr/tests/rest/test_permissions.py b/akvo/rsr/tests/rest/test_permissions.py index 58e7af81b7..e9884afd4b 100644 --- a/akvo/rsr/tests/rest/test_permissions.py +++ b/akvo/rsr/tests/rest/test_permissions.py @@ -9,7 +9,6 @@ import collections import re -import unittest from django.conf import settings from django.contrib.auth import get_user_model @@ -19,6 +18,7 @@ from akvo.rest.viewsets import PublicProjectViewSet, ReadOnlyPublicProjectViewSet from akvo.rsr import models as M +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.utils import check_auth_groups @@ -211,11 +211,12 @@ def setUpClass(cls): latitude=update.id, longitude=update.id) # indicator period data - data = M.IndicatorPeriodData.objects.create(period=period, user=user) + data = M.IndicatorPeriodData.objects.create(period=period, user=user, status=M.IndicatorPeriodData.STATUS_APPROVED_CODE) # disaggregation M.Disaggregation.objects.create(update=data, dimension_value=dimension_value) # indicator period data comment M.IndicatorPeriodDataComment.objects.create(data=data, user=user) + aggregate(period) # PartnerSite M.PartnerSite.objects.create(organisation=organisation, @@ -613,7 +614,6 @@ def set_publishing_status(project, status): publishing_status.status = status publishing_status.save() - @unittest.skip('aggregation behaviour refactoring') def test_admin(self): for user in User.objects.filter(is_admin=True): for queryset, project_relation, count in self.iter_queryset('admin'): @@ -622,7 +622,6 @@ def test_admin(self): ) self.assertPermissions(user, count, filtered_queryset) - @unittest.skip('aggregation behaviour refactoring') def test_anonymous(self): user = AnonymousUser() for queryset, project_relation, count in self.iter_queryset('anonymous'): @@ -631,7 +630,6 @@ def test_anonymous(self): ) self.assertPermissions(user, count, filtered_queryset) - @unittest.skip('aggregation behaviour refactoring') def test_logged_in_user(self): m_e = Group.objects.get(name='M&E Managers') p_e = Group.objects.get(name='Project Editors') diff --git a/akvo/rsr/tests/rest/test_project_overview.py b/akvo/rsr/tests/rest/test_project_overview.py index cf2131bf81..cef70eea94 100644 --- a/akvo/rsr/tests/rest/test_project_overview.py +++ b/akvo/rsr/tests/rest/test_project_overview.py @@ -4,16 +4,15 @@ # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. -import unittest from unittest.mock import patch from akvo.rsr.models import Result, Indicator, IndicatorPeriod, IndicatorPeriodData from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.models.result.utils import QUANTITATIVE, QUALITATIVE, PERCENTAGE_MEASURE +from akvo.rsr.usecases.period_update_aggregation import aggregate class QuantitativeUnitAggregationTestCase(BaseTestCase): - @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -32,7 +31,6 @@ def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self) final_value = response.data['indicators'][0]['periods'][0]['actual_value'] self.assertEqual(final_value, 1 + 2) - @unittest.skip('aggregation behaviour refactoring') def test_updates_from_every_level_of_hierarchy_are_calculated_for_final_value(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -148,7 +146,6 @@ def test_handle_update_with_null_value(self): class QuantitativePercentageAggregationTestCase(BaseTestCase): - @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -312,7 +309,6 @@ def test_handle_update_with_null_value(self): class QualitativeScoresAggregationTestCase(BaseTestCase): - @unittest.skip('aggregation behaviour refactoring') def test_updates_from_contributing_projects_are_aggregated_to_lead_project(self): url = ProjectHierarchyFixtureBuilder(self)\ .with_hierarchy({ @@ -567,6 +563,7 @@ def _handle_updates(self, project_map, user): score_index=update.get('score_index', None), status=update.get('status', 'A') ) + aggregate(period) class ProjectMapHelper(object): diff --git a/akvo/rsr/tests/results_framework/test_aggregation.py b/akvo/rsr/tests/results_framework/test_aggregation.py index 321cdfcb90..fc8b6a21f4 100644 --- a/akvo/rsr/tests/results_framework/test_aggregation.py +++ b/akvo/rsr/tests/results_framework/test_aggregation.py @@ -15,6 +15,7 @@ IndicatorPeriodData, User, RelatedProject) from akvo.rsr.models.result.utils import (calculate_percentage, MultipleUpdateError) +from akvo.rsr.usecases.period_update_aggregation import aggregate from django.test import TestCase @@ -147,28 +148,30 @@ def test_should_update_actual_value_with_update_data(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(str(actual_value + increment), period.actual_value) - @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_update_numeric_data(self): # Given increment = 2 self.create_indicator_period_update(value=increment) + aggregate(self.period) # When self.create_indicator_period_update(value=increment) + aggregate(self.period) # Then period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(increment * 2), Decimal(period.actual_value)) - @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_update_str_negative_data(self): # Given original = 5 increment = -2 self.create_indicator_period_update(value=str(original)) + aggregate(self.period) # When self.create_indicator_period_update(value=str(increment)) + aggregate(self.period) # Then period = IndicatorPeriod.objects.get(id=self.period.id) @@ -190,7 +193,6 @@ def test_should_replace_with_non_numeric_update_data(self): # Single child tests - @unittest.skip('aggregation behaviour refactoring') def test_should_copy_child_period_value(self): # Given value = 5 @@ -205,12 +207,12 @@ def test_should_copy_child_period_value(self): value=child_value, indicator_period=child_indicator_period ) + aggregate(child_indicator_period) # Then period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(child_value), Decimal(period.actual_value)) - @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_child_period_value(self): # Given self.parent_project.aggregate_children = False @@ -227,6 +229,7 @@ def test_should_not_aggregate_child_period_value(self): value=child_value, indicator_period=child_indicator_period ) + aggregate(child_indicator_period) # Then period = IndicatorPeriod.objects.get(id=self.period.id) @@ -234,7 +237,6 @@ def test_should_not_aggregate_child_period_value(self): # Multiple children tests - @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_child_indicators_values(self): # Given child_project_2 = self.create_child_project('Child project 2') @@ -246,13 +248,14 @@ def test_should_aggregate_child_indicators_values(self): # When self.create_indicator_period_update(value=value, indicator_period=child_indicator_period) + aggregate(child_indicator_period) self.create_indicator_period_update(value=value, indicator_period=child_indicator_period_2) + aggregate(child_indicator_period_2) # Then period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(value * 2), Decimal(period.actual_value)) - @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_excluded_child_period_values(self): # Given child_project_2 = self.create_child_project('Child project 2') @@ -266,7 +269,9 @@ def test_should_not_aggregate_excluded_child_period_values(self): # When self.create_indicator_period_update(value=value, indicator_period=child_indicator_period) + aggregate(child_indicator_period) self.create_indicator_period_update(value=value, indicator_period=child_indicator_period_2) + aggregate(child_indicator_period_2) # Then period = IndicatorPeriod.objects.get(id=self.period.id) @@ -335,7 +340,6 @@ def test_should_replace_with_non_numeric_update_data(self): # Single child tests - @unittest.skip('aggregation behaviour refactoring') def test_should_copy_child_period_value(self): # Given numerator = 5 @@ -357,6 +361,7 @@ def test_should_copy_child_period_value(self): denominator=child_denominator, indicator_period=child_indicator_period ) + aggregate(child_indicator_period) # Then period = IndicatorPeriod.objects.get(id=self.period.id) @@ -394,7 +399,6 @@ def test_should_not_aggregate_child_period_value(self): # Multiple children tests - @unittest.skip('aggregation behaviour refactoring') def test_should_aggregate_child_indicators_values(self): # Given numerator = 5 @@ -416,11 +420,13 @@ def test_should_aggregate_child_indicators_values(self): denominator=child_denominator, indicator_period=child_period ) + aggregate(child_period) self.create_indicator_period_update( numerator=child_numerator, denominator=child_denominator, indicator_period=child_period_2 ) + aggregate(child_period_2) # Then period = IndicatorPeriod.objects.get(id=self.period.id) @@ -429,7 +435,6 @@ def test_should_aggregate_child_indicators_values(self): self.assertDecimalEqual(child_numerator * 2, period.numerator) self.assertDecimalEqual(child_denominator * 2, period.denominator) - @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_excluded_child_period_values(self): # Given child_project_2 = self.create_child_project('Child project 2') @@ -448,11 +453,13 @@ def test_should_not_aggregate_excluded_child_period_values(self): denominator=child_denominator, indicator_period=child_period ) + aggregate(child_period) self.create_indicator_period_update( numerator=child_numerator, denominator=child_denominator, indicator_period=child_period_2 ) + aggregate(child_period_2) # Then period = IndicatorPeriod.objects.get(id=self.period.id) diff --git a/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py b/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py index 6c327a3b58..34dd249327 100644 --- a/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py +++ b/akvo/rsr/tests/results_framework/test_disaggregation_aggregation.py @@ -1,10 +1,10 @@ import datetime -import unittest from akvo.rsr.models import ( Result, Indicator, IndicatorPeriod, IndicatorDimensionName, IndicatorDimensionValue, IndicatorPeriodData) +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.tests.base import BaseTestCase from . import util @@ -29,7 +29,6 @@ def setUp(self): IndicatorDimensionValue.objects.create(name=category, value=self.type2) self.indicator.dimension_names.add(category) - @unittest.skip('aggregation behaviour refactoring') def test_aggregate_to_period(self): # Given disaggregations = util.get_disaggregations(self.project) @@ -46,6 +45,7 @@ def test_aggregate_to_period(self): ], value=50 ) + aggregate(self.period) util.create_period_update( period=self.period, user=self.user, @@ -55,6 +55,7 @@ def test_aggregate_to_period(self): ], value=30 ) + aggregate(self.period) # Then self.assertEqual(self.period.disaggregations.count(), 2) @@ -67,7 +68,6 @@ def test_aggregate_to_period(self): 30 + 15 ) - @unittest.skip('aggregation behaviour refactoring') def test_aggregate_percentages_to_period(self): # Given disaggregations = util.get_disaggregations(self.project) @@ -85,6 +85,7 @@ def test_aggregate_percentages_to_period(self): numerator=50, denominator=100 ) + aggregate(self.period) util.create_period_update( period=self.period, user=self.user, @@ -95,6 +96,7 @@ def test_aggregate_percentages_to_period(self): numerator=30, denominator=100 ) + aggregate(self.period) # Then self.assertEqual(self.period.disaggregations.count(), 2) @@ -107,7 +109,6 @@ def test_aggregate_percentages_to_period(self): self.assertEqual(period_disaggregation2.numerator, 30 + 15) self.assertEqual(period_disaggregation2.denominator, 50 + 50) - @unittest.skip('aggregation behaviour refactoring') def test_aggregate_child_period_to_parent(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -118,7 +119,7 @@ def test_aggregate_child_period_to_parent(self): child2 = self.create_contributor("Child 2", self.project) child2_period = util.get_periods(child2).first() - child2_disaggregations = util.get_disaggregations(child1) + child2_disaggregations = util.get_disaggregations(child2) child2_type1 = child2_disaggregations.filter(value=self.type1).first() child2_type2 = child2_disaggregations.filter(value=self.type2).first() @@ -127,10 +128,12 @@ def test_aggregate_child_period_to_parent(self): {'type': child1_type1, 'value': 20}, {'type': child1_type2, 'value': 30}, ]) + aggregate(child1_period) util.create_period_update(child2_period, self.user, disaggregations=[ {'type': child2_type1, 'value': 15}, {'type': child2_type2, 'value': 15}, ]) + aggregate(child2_period) # Then self.assertEqual( @@ -142,7 +145,6 @@ def test_aggregate_child_period_to_parent(self): 30 + 15 ) - @unittest.skip('aggregation behaviour refactoring') def test_should_not_aggregate_unapproved_updates(self): # Given child = self.create_contributor("Child", self.project) @@ -161,6 +163,7 @@ def test_should_not_aggregate_unapproved_updates(self): ], status=IndicatorPeriodData.STATUS_PENDING_CODE ) + aggregate(child_period) # Then self.assertEqual(self.period.disaggregations.count(), 2) @@ -183,7 +186,6 @@ def test_should_not_aggregate_unapproved_updates(self): None ) - @unittest.skip('aggregation behaviour refactoring') def test_project_with_aggregate_to_parent_off(self): # Given child = self.create_contributor("Child", self.project) @@ -204,6 +206,7 @@ def test_project_with_aggregate_to_parent_off(self): {'type': child_type2, 'value': 30}, ] ) + aggregate(child_period) # Then self.assertEqual(self.period.disaggregations.count(), 0) @@ -218,7 +221,6 @@ def test_project_with_aggregate_to_parent_off(self): 30 ) - @unittest.skip('aggregation behaviour refactoring') def test_project_with_aggregate_children_off(self): # Given self.project.aggregate_children = False @@ -239,6 +241,7 @@ def test_project_with_aggregate_children_off(self): {'type': child_type2, 'value': 30}, ] ) + aggregate(child_period) # Then self.assertEqual(self.period.disaggregations.count(), 0) @@ -253,7 +256,6 @@ def test_project_with_aggregate_children_off(self): 30 ) - @unittest.skip('aggregation behaviour refactoring') def test_aggregate_multi_level_hierarchy(self): # Given child = self.create_contributor("Child", self.project) @@ -270,10 +272,12 @@ def test_aggregate_multi_level_hierarchy(self): {'type': grandchild_type1, 'value': 20}, {'type': grandchild_type2, 'value': 30}, ]) + aggregate(grandchild_period) util.create_period_update(grandchild_period, self.user, disaggregations=[ {'type': grandchild_type1, 'value': 15}, {'type': grandchild_type2, 'value': 15}, ]) + aggregate(grandchild_period) # Then self.assertEqual(grandchild_period.disaggregations.count(), 2) @@ -306,7 +310,6 @@ def test_aggregate_multi_level_hierarchy(self): 30 + 15 ) - @unittest.skip('aggregation behaviour refactoring') def test_aggregate_multiple_periods(self): # Given period2 = IndicatorPeriod.objects.create( @@ -327,10 +330,12 @@ def test_aggregate_multiple_periods(self): {'type': child_type1, 'value': 20}, {'type': child_type2, 'value': 30}, ]) + aggregate(child_period1) util.create_period_update(child_period2, self.user, disaggregations=[ {'type': child_type1, 'value': 15}, {'type': child_type2, 'value': 15}, ]) + aggregate(child_period2) # Then self.assertEqual(child_period1.disaggregations.count(), 2) @@ -371,7 +376,6 @@ def test_aggregate_multiple_periods(self): 15 ) - @unittest.skip('aggregation behaviour refactoring') def test_sum_local_and_aggregated_updates(self): # Given child = self.create_contributor("Child", self.project) @@ -391,10 +395,12 @@ def test_sum_local_and_aggregated_updates(self): {'type': grandchild_type1, 'value': 20}, {'type': grandchild_type2, 'value': 30}, ]) + aggregate(grandchild_period) util.create_period_update(child_period, self.user, disaggregations=[ {'type': child_type1, 'value': 15}, {'type': child_type2, 'value': 15}, ]) + aggregate(child_period) # Then self.assertEqual(grandchild_period.disaggregations.count(), 2) @@ -427,7 +433,6 @@ def test_sum_local_and_aggregated_updates(self): 30 + 15 ) - @unittest.skip('aggregation behaviour refactoring') def test_amend_on_the_lowest_level_case1(self): # Given child = self.create_contributor("Child", self.project) @@ -449,17 +454,21 @@ def test_amend_on_the_lowest_level_case1(self): {'type': grandchild1_type1, 'value': 20}, {'type': grandchild1_type2, 'value': 30}, ]) + aggregate(grandchild1_period) target_amend_update = util.create_period_update(grandchild1_period, self.user, disaggregations=[ {'type': grandchild1_type1, 'value': 15}, {'type': grandchild1_type2, 'value': 15}, ]) + aggregate(grandchild1_period) util.create_period_update(grandchild2_period, self.user, disaggregations=[ {'type': grandchild2_type1, 'value': 30}, {'type': grandchild2_type2, 'value': 30}, ]) + aggregate(grandchild2_period) # When util.amend_disaggregation_update(target_amend_update, self.type1, 20) + aggregate(grandchild1_period) # Then self.assertEqual(self.period.disaggregations.count(), 2) @@ -482,7 +491,6 @@ def test_amend_on_the_lowest_level_case1(self): 30 + 15 + 30 ) - @unittest.skip('aggregation behaviour refactoring') def test_amend_on_the_lowest_level_case2(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -507,17 +515,21 @@ def test_amend_on_the_lowest_level_case2(self): {'type': grandchild1_type1, 'value': 20}, {'type': grandchild1_type2, 'value': 30}, ]) + aggregate(grandchild1_period) target_amend_update = util.create_period_update(grandchild1_period, self.user, disaggregations=[ {'type': grandchild1_type1, 'value': 15}, {'type': grandchild1_type2, 'value': 15}, ]) + aggregate(grandchild1_period) util.create_period_update(grandchild2_period, self.user, disaggregations=[ {'type': grandchild2_type1, 'value': 30}, {'type': grandchild2_type2, 'value': 30}, ]) + aggregate(grandchild2_period) # When util.amend_disaggregation_update(target_amend_update, self.type1, 20) + aggregate(grandchild1_period) # Then self.assertEqual(self.period.disaggregations.count(), 2) @@ -550,7 +562,6 @@ def test_amend_on_the_lowest_level_case2(self): 30 ) - @unittest.skip('aggregation behaviour refactoring') def test_delete_on_the_lowest_level(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -572,17 +583,21 @@ def test_delete_on_the_lowest_level(self): {'type': grandchild1_type1, 'value': 20}, {'type': grandchild1_type2, 'value': 30}, ]) + aggregate(grandchild1_period) target_update = util.create_period_update(grandchild1_period, self.user, disaggregations=[ {'type': grandchild1_type1, 'value': 15}, {'type': grandchild1_type2, 'value': 15}, ]) + aggregate(grandchild1_period) util.create_period_update(grandchild2_period, self.user, disaggregations=[ {'type': grandchild2_type1, 'value': 30}, {'type': grandchild2_type2, 'value': 30}, ]) + aggregate(grandchild2_period) # When target_update.delete() + aggregate(grandchild1_period) # Then self.assertEqual(self.period.disaggregations.count(), 2) diff --git a/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py b/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py index 42038a852a..90034e15a1 100644 --- a/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py +++ b/akvo/rsr/tests/results_framework/test_disaggregation_contribution.py @@ -8,10 +8,10 @@ """ import datetime -import unittest from akvo.rsr.models import ( Result, Indicator, IndicatorPeriod, IndicatorDimensionName, IndicatorDimensionValue) +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.tests.base import BaseTestCase from . import util @@ -34,7 +34,6 @@ def setUp(self): IndicatorDimensionValue.objects.create(name=category, value=self.type) indicator.dimension_names.add(category) - @unittest.skip('aggregation behaviour refactoring') def test_no_contribution(self): # Given type = util.get_disaggregations(self.project).filter(value=self.type).first() @@ -43,16 +42,17 @@ def test_no_contribution(self): util.create_period_update( period=self.period, user=self.user, value=1, disaggregations=[{'type': type, 'value': 20}]) + aggregate(self.period) util.create_period_update( period=self.period, user=self.user, value=1, disaggregations=[{'type': type, 'value': 15}]) + aggregate(self.period) # Then self.assertEqual( util.get_disaggregation_contributors(self.period, self.type).count(), 0) - @unittest.skip('aggregation behaviour refactoring') def test_disaggregation_contribution_from_child_to_parent(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -67,9 +67,11 @@ def test_disaggregation_contribution_from_child_to_parent(self): util.create_period_update( period=child1_period, user=self.user, value=1, disaggregations=[{'type': child1_type, 'value': 20}]) + aggregate(child1_period) util.create_period_update( period=child2_period, user=self.user, value=1, disaggregations=[{'type': child2_type, 'value': 15}]) + aggregate(child2_period) # Then self.assertEqual( @@ -82,7 +84,6 @@ def test_disaggregation_contribution_from_child_to_parent(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 15) - @unittest.skip('aggregation behaviour refactoring') def test_multi_level_disaggregation_contribution(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -103,9 +104,11 @@ def test_multi_level_disaggregation_contribution(self): util.create_period_update( period=grandchild1_period, user=self.user, value=1, disaggregations=[{'type': grandchild1_type, 'value': 20}]) + aggregate(grandchild1_period) util.create_period_update( period=grandchild2_period, user=self.user, value=1, disaggregations=[{'type': grandchild2_type, 'value': 15}]) + aggregate(grandchild2_period) # Then self.assertEqual( @@ -118,7 +121,6 @@ def test_multi_level_disaggregation_contribution(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 15) - @unittest.skip('aggregation behaviour refactoring') def test_amend_disaggregation_contributions(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -138,12 +140,15 @@ def test_amend_disaggregation_contributions(self): util.create_period_update( period=grandchild1_period, user=self.user, value=1, disaggregations=[{'type': grandchild1_type, 'value': 20}]) + aggregate(grandchild1_period) target_amend_update = util.create_period_update( period=grandchild2_period, user=self.user, value=1, disaggregations=[{'type': grandchild2_type, 'value': 15}]) + aggregate(grandchild2_period) # When util.amend_disaggregation_update(target_amend_update, self.type, 30) + aggregate(grandchild2_period) # Then self.assertEqual( @@ -156,7 +161,6 @@ def test_amend_disaggregation_contributions(self): util.get_disaggregation_contributors(self.period, self.type, child2).value, 30) - @unittest.skip('aggregation behaviour refactoring') def test_delete_period_update_contributions(self): # Given child1 = self.create_contributor("Child 1", self.project) @@ -177,12 +181,15 @@ def test_delete_period_update_contributions(self): util.create_period_update( period=grandchild1_period, user=self.user, value=1, disaggregations=[{'type': grandchild1_type, 'value': 20}]) + aggregate(grandchild1_period) target_update = util.create_period_update( period=grandchild2_period, user=self.user, value=1, disaggregations=[{'type': grandchild2_type, 'value': 15}]) + aggregate(grandchild2_period) # When target_update.delete() + aggregate(grandchild2_period) # Then self.assertEqual( diff --git a/akvo/rsr/tests/results_framework/test_results_framework.py b/akvo/rsr/tests/results_framework/test_results_framework.py index bdcb00dac5..25463a9a7f 100644 --- a/akvo/rsr/tests/results_framework/test_results_framework.py +++ b/akvo/rsr/tests/results_framework/test_results_framework.py @@ -17,6 +17,7 @@ RelatedProject, IndicatorDimensionName, IndicatorDimensionValue, DefaultPeriod, Project) from akvo.rsr.models.related_project import MultipleParentsDisallowed, ParentChangeDisallowed from akvo.rsr.models.result.utils import QUALITATIVE +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.tests.base import BaseTestCase @@ -521,7 +522,6 @@ def test_child_dimension_value_delete_prevented(self): with self.assertRaises(PermissionDenied): child_dimension_name.dimension_values.last().delete() - @unittest.skip('aggregation behaviour refactoring') def test_update(self): """ Test if placing updates will update the actual value of the period. @@ -531,10 +531,12 @@ def test_update(self): period=self.period, value="10" ) + aggregate(self.period) self.assertEqual(self.period.actual_value, "") indicator_update.status = "A" indicator_update.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "10.00") indicator_update_2 = IndicatorPeriodData.objects.create( @@ -542,13 +544,14 @@ def test_update(self): period=self.period, value="5" ) + aggregate(self.period) self.assertEqual(self.period.actual_value, "10.00") indicator_update_2.status = "A" indicator_update_2.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "15.00") - @unittest.skip('aggregation behaviour refactoring') def test_edit_and_delete_updates(self): """ Test if editing or deleting updates will update the actual value of the period. @@ -558,20 +561,23 @@ def test_edit_and_delete_updates(self): period=self.period, value="10" ) + aggregate(self.period) self.assertEqual(self.period.actual_value, "") indicator_update.status = "A" indicator_update.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "10.00") indicator_update.value = "11" indicator_update.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "11.00") indicator_update.delete() - self.assertEqual(self.period.actual_value, "0") + aggregate(self.period) + self.assertEqual(self.period.actual_value, "") - @unittest.skip('aggregation behaviour refactoring') def test_update_on_child(self): """ Test if placing an update on the child project will update the actual value of the period, @@ -582,10 +588,12 @@ def test_update_on_child(self): period=self.period, value="10" ) + aggregate(self.period) self.assertEqual(self.period.actual_value, "") indicator_update.status = "A" indicator_update.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "10.00") child_period = IndicatorPeriod.objects.filter( @@ -596,17 +604,18 @@ def test_update_on_child(self): period=child_period, value=15 ) + aggregate(child_period) self.assertEqual(child_period.actual_value, "") indicator_update_2.status = "A" indicator_update_2.save() + aggregate(child_period) self.assertEqual(child_period.actual_value, "15.00") parent_period = IndicatorPeriod.objects.filter( indicator__result__project=self.parent_project).first() self.assertEqual(parent_period.actual_value, "25.00") - @unittest.skip('aggregation behaviour refactoring') def test_update_without_aggregations(self): """ Test if placing an update on the child project without an aggregation will not update the @@ -617,10 +626,12 @@ def test_update_without_aggregations(self): period=self.period, value=10 ) + aggregate(self.period) self.assertEqual(self.period.actual_value, "") indicator_update.status = "A" indicator_update.save() + aggregate(self.period) self.assertEqual(self.period.actual_value, "10.00") parent_project = self.period.indicator.result.project @@ -635,17 +646,18 @@ def test_update_without_aggregations(self): period=child_period, value=15 ) + aggregate(child_period) self.assertEqual(child_period.actual_value, "") indicator_update_2.status = "A" indicator_update_2.save() + aggregate(child_period) self.assertEqual(child_period.actual_value, "15.00") parent_period = IndicatorPeriod.objects.filter( indicator__result__project=self.parent_project).first() self.assertEqual(parent_period.actual_value, "10.00") - @unittest.skip('aggregation behaviour refactoring') def test_updates_with_percentages(self): """ Test if placing an update on two child projects will give the average of the two in the @@ -680,10 +692,12 @@ def test_updates_with_percentages(self): numerator="4", denominator="6", ) + aggregate(child_period) self.assertEqual(child_period.actual_value, "") indicator_update.status = "A" indicator_update.save() + aggregate(child_period) self.assertEqual(child_period.actual_value, "66.67") self.assertEqual(parent_indicator.periods.first().actual_value, "66.67") @@ -693,10 +707,12 @@ def test_updates_with_percentages(self): numerator="2", denominator="4", ) + aggregate(child_2_period) self.assertEqual(child_2_period.actual_value, "") indicator_update_2.status = "A" indicator_update_2.save() + aggregate(child_2_period) self.assertEqual(child_2_period.actual_value, "50.00") self.assertEqual(parent_indicator.periods.first().actual_value, "60.00") From 12117911aba99dc183c313bde562b0d6d5315de2 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Wed, 19 Oct 2022 09:54:08 +0700 Subject: [PATCH 15/59] [#5140] Cleanup and remove dead codes --- ...ink_results_framework_to_parent_project.py | 3 +- .../commands/recalculate_periods.py | 3 +- akvo/rsr/models/project.py | 73 +------ akvo/rsr/models/result/disaggregation.py | 21 -- akvo/rsr/models/result/indicator_period.py | 179 ------------------ .../models/result/indicator_period_data.py | 2 - 6 files changed, 6 insertions(+), 275 deletions(-) diff --git a/akvo/rsr/management/commands/link_results_framework_to_parent_project.py b/akvo/rsr/management/commands/link_results_framework_to_parent_project.py index 9022e12807..b6eb9cb33f 100644 --- a/akvo/rsr/management/commands/link_results_framework_to_parent_project.py +++ b/akvo/rsr/management/commands/link_results_framework_to_parent_project.py @@ -7,6 +7,7 @@ from django.core.management.base import BaseCommand from akvo.rsr.models import Project +from akvo.rsr.usecases.period_update_aggregation import aggregate class Command(BaseCommand): @@ -32,4 +33,4 @@ def handle(self, *args, **options): parent_period = parent_indicator.periods.get(period_start=period.period_start, period_end=period.period_end) period.parent_period = parent_period period.save(update_fields=['parent_period']) - parent_period.recalculate_period() + aggregate(parent_period) diff --git a/akvo/rsr/management/commands/recalculate_periods.py b/akvo/rsr/management/commands/recalculate_periods.py index 00ad46e934..62489f6141 100644 --- a/akvo/rsr/management/commands/recalculate_periods.py +++ b/akvo/rsr/management/commands/recalculate_periods.py @@ -7,6 +7,7 @@ from django.core.management.base import BaseCommand from django.db.models import Count +from akvo.rsr.usecases.period_update_aggregation import aggregate from ...models import IndicatorPeriod @@ -41,4 +42,4 @@ def handle(self, *args, **options): for period in periods: if verbosity > 1: print('Recalculating period {}'.format(period.id)) - period.recalculate_period() + aggregate(period) diff --git a/akvo/rsr/models/project.py b/akvo/rsr/models/project.py index e17cc8925d..ad37f83431 100644 --- a/akvo/rsr/models/project.py +++ b/akvo/rsr/models/project.py @@ -5,7 +5,7 @@ For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. """ import logging -from decimal import Decimal, InvalidOperation +from decimal import Decimal import itertools import urllib.parse @@ -44,7 +44,6 @@ from ..fields import ProjectLimitedTextField, ValidXMLCharField, ValidXMLTextField from ..mixins import TimestampsMixin -from .result import IndicatorPeriod from .model_querysets.project import ProjectQuerySet from .partnership import Partnership from .project_update import ProjectUpdate @@ -502,7 +501,7 @@ def save(self, *args, **kwargs): if not self.iati_activity_id: self.iati_activity_id = None - orig, orig_aggregate_children, orig_aggregate_to_parent = None, None, None + orig = None if self.pk: # If the project is being deleted, don't allow saving it if self.pk in DELETION_SET: @@ -530,26 +529,8 @@ def save(self, *args, **kwargs): descendants = self.descendants() descendants.exclude(pk=self.pk).update(targets_at=self.targets_at) - orig_aggregate_children = orig.aggregate_children - orig_aggregate_to_parent = orig.aggregate_to_parent - super(Project, self).save(*args, **kwargs) - if orig: - # Update aggregation from children - if self.aggregate_children != orig_aggregate_children: - for period in IndicatorPeriod.objects.filter(indicator__result__project_id=self.pk): - if self.aggregate_children: - period.recalculate_period() - else: - period.recalculate_period(only_self=True) - - # Update aggregation to parent - if self.aggregate_to_parent != orig_aggregate_to_parent: - for period in IndicatorPeriod.objects.filter(indicator__result__project_id=self.pk): - if period.parent_period: - period.parent_period.recalculate_period() - def clean(self): # Don't allow a start date before an end date if self.date_start_planned and self.date_end_planned and \ @@ -1654,56 +1635,6 @@ def indicator_labels(self): def has_indicator_labels(self): return self.indicator_labels().count() > 0 - def toggle_aggregate_children(self, aggregate): - """ - If aggregation to children is turned off, - - :param aggregate; Boolean, indicating if aggregation is turned on (True) or off (False) - """ - for result in self.results.all(): - for indicator in result.indicators.all(): - if indicator.is_parent_indicator(): - for period in indicator.periods.all(): - if indicator.measure == '2': - self.update_parents(period, period.child_periods_average(), 1) - else: - sign = 1 if aggregate else -1 - self.update_parents(period, period.child_periods_sum(), sign) - - def toggle_aggregate_to_parent(self, aggregate): - """ Add/subtract child indicator period values from parent if aggregation is toggled """ - for result in self.results.all(): - for indicator in result.indicators.all(): - if indicator.is_child_indicator(): - for period in indicator.periods.all(): - parent = period.parent_period - if parent and period.actual_value: - if indicator.measure == '2': - self.update_parents(parent, parent.child_periods_average(), 1) - else: - sign = 1 if aggregate else -1 - self.update_parents(parent, period.actual_value, sign) - - def update_parents(self, update_period, difference, sign): - """ Update parent indicator periods if they exist and allow aggregation """ - try: - if update_period.indicator.measure == '2': - update_period.actual_value = str(Decimal(difference)) - else: - update_period.actual_value = str( - Decimal(update_period.actual_value) + sign * Decimal(difference)) - update_period.save() - - parent_period = update_period.parent_period - if parent_period and parent_period.indicator.result.project.aggregate_children: - if update_period.indicator.measure == '2': - self.update_parents(parent_period, parent_period.child_periods_average(), 1) - else: - self.update_parents(parent_period, difference, sign) - - except (InvalidOperation, TypeError): - pass - def update_use_project_roles(self): if not self.reporting_org: return diff --git a/akvo/rsr/models/result/disaggregation.py b/akvo/rsr/models/result/disaggregation.py index 6b2036f846..30aa7e4d32 100644 --- a/akvo/rsr/models/result/disaggregation.py +++ b/akvo/rsr/models/result/disaggregation.py @@ -70,27 +70,6 @@ def update_incomplete_data(self): self.siblings().update(incomplete_data=incomplete_data) -# @receiver([signals.post_save, signals.post_delete], sender=Disaggregation) -# def aggregate_period_disaggregation_up_to_parent_hierarchy(sender, **kwargs): -# -# # Disable signal handler when loading fixtures -# if kwargs.get('raw', False): -# return -# -# from .disaggregation_aggregation import DisaggregationAggregation -# from .indicator_period_disaggregation import IndicatorPeriodDisaggregation -# -# disaggregation = kwargs['instance'] -# disaggregation_aggregation = DisaggregationAggregation( -# Disaggregation.objects, -# IndicatorPeriodDisaggregation.objects -# ) -# disaggregation_aggregation.aggregate( -# disaggregation.update.period, -# disaggregation.dimension_value -# ) - - @receiver(signals.post_save, sender=Disaggregation) def mark_incomplete_disaggregations(sender, **kwargs): """Mark disaggregations as incomplete if they don't add up to the period value.""" diff --git a/akvo/rsr/models/result/indicator_period.py b/akvo/rsr/models/result/indicator_period.py index 36f30d4241..0cd6e954d6 100644 --- a/akvo/rsr/models/result/indicator_period.py +++ b/akvo/rsr/models/result/indicator_period.py @@ -96,7 +96,6 @@ def __str__(self): return period_unicode def save(self, *args, **kwargs): - # actual_value_changed = False new_period = not self.pk if ( @@ -107,12 +106,6 @@ def save(self, *args, **kwargs): percentage = calculate_percentage(self.numerator, self.denominator) self.actual_value = str(percentage) - # if not new_period: - # # Check if the actual value has changed - # orig_period = IndicatorPeriod.objects.get(pk=self.pk) - # if orig_period.actual_value != self.actual_value: - # actual_value_changed = True - super(IndicatorPeriod, self).save(*args, **kwargs) child_indicators = self.indicator.child_indicators.select_related( @@ -126,13 +119,6 @@ def save(self, *args, **kwargs): else: child_indicator.result.project.update_period(child_indicator, self) - # If the actual value has changed, the period has a parent period and aggregations are on, - # then the the parent should be updated as well - # if actual_value_changed and self.is_child_period() and \ - # self.parent_period.indicator.result.project.aggregate_children and \ - # self.indicator.result.project.aggregate_to_parent: - # self.parent_period.recalculate_period() - def clean(self): validation_errors = {} @@ -172,104 +158,6 @@ def clean(self): if validation_errors: raise ValidationError(validation_errors) - def recalculate_period(self, save=True, only_self=False): - """ - Re-calculate the values of all updates from the start. This will prevent strange values, - for example when an update is deleted or edited after it has been approved. - - :param save; Boolean, saves actual value to period if True - :param only_self; Boolean, to take into account if this is a parent or just re-calculate - this period only - :return Actual value of period - """ - - # If this period is a parent period, the sum or average of the children - # should be re-calculated - if not only_self and self.is_parent_period() and \ - self.indicator.result.project.aggregate_children: - return self.recalculate_children(save) - - prev_val = '0' - if self.indicator.measure == PERCENTAGE_MEASURE: - prev_num = '0' - prev_den = '0' - - # For every approved update, add up the new value (if possible) - for update in self.data.filter(status='A').order_by('created_at'): - if self.indicator.measure == PERCENTAGE_MEASURE: - update.period_numerator = prev_num - update.period_denominator = prev_den - update.period_actual_value = prev_val - update.save(recalculate=False) - - if update.value is None: - continue - - try: - # Try to add up the update to the previous actual value - if self.indicator.measure == PERCENTAGE_MEASURE: - prev_num = str(Decimal(prev_num) + Decimal(update.numerator)) - prev_den = str(Decimal(prev_den) + Decimal(update.denominator)) - prev_val = str(calculate_percentage(float(prev_num), float(prev_den))) - else: - prev_val = str(Decimal(prev_val) + Decimal(update.value)) - except (InvalidOperation, TypeError): - # If not possible, the update data or previous value is a normal string - if self.indicator.measure == PERCENTAGE_MEASURE: - prev_num = update.numerator - prev_den = update.denominator - prev_val = update.value - - # For every non-approved update, set the value to the current value - for update in self.data.exclude(status='A'): - update.period_actual_value = prev_val - if self.indicator.measure == PERCENTAGE_MEASURE: - update.period_numerator = prev_num - update.period_denominator = prev_den - update.save(recalculate=False) - - # Special case: only_self and no data should give an empty string instead of '0' - if only_self and not self.data.exists(): - prev_val = '' - # FIXME: Do we need a special case here with numerator and denominator??? - - # Finally, update the actual value of the period itself - if save: - self.actual_value = prev_val - if self.indicator.measure == PERCENTAGE_MEASURE: - self.numerator = prev_num - self.denominator = prev_den - self.save() - - # Return the actual value of the period itself - return prev_val - - def recalculate_children(self, save=True): - """ - Re-calculate the actual value of this period based on the actual values of the child - periods. - - In case the measurement is 'Percentage', it should be an average of all child periods. - Otherwise, the child period values can just be added up. - - :param save; Boolean, saves to period if True - :return Actual value of period - """ - if self.indicator.measure == PERCENTAGE_MEASURE: - numerator, denominator = self.child_periods_percentage() - new_value = calculate_percentage(numerator, denominator) - else: - new_value = self.child_periods_sum(include_self=True) - - if save: - self.actual_value = new_value - if self.indicator.measure == PERCENTAGE_MEASURE: - self.numerator = numerator - self.denominator = denominator - self.save() - - return new_value - def update_actual_comment(self, save=True): """ Set the actual comment to the text of the latest approved update. @@ -335,12 +223,6 @@ def is_child_period(self): """ return bool(self.parent_period) - def is_parent_period(self): - """ - Indicates whether this result has child periods linked to it. - """ - return self.child_periods.count() > 0 - def can_save_update(self, update_id=None): """Return True if an update can be created/updated on the indicator period. @@ -357,67 +239,6 @@ def can_save_update(self, update_id=None): or self.data.exclude(id=update_id).count() == 0 ) - def child_periods_with_data(self, only_aggregated=False): - """ - Returns the child indicator periods with numeric values - """ - children_with_data = [] - for child in self.child_periods.all(): - try: - Decimal(child.actual_value) - children_with_data += [child.pk] - except (InvalidOperation, TypeError): - pass - child_periods = self.child_periods.filter(pk__in=children_with_data) - if only_aggregated: - child_periods = child_periods.filter( - indicator__result__project__aggregate_to_parent=True - ) - return child_periods - - # TODO: refactor child_periods_sum() and child_periods_with_data(), - # they use each other in very inefficient ways I think - def child_periods_sum(self, include_self=False): - """ - Returns the sum of child indicator periods. - - :param include_self; Boolean to include the updates on the period itself, as well as its' - children - :return String of the sum - """ - period_sum = 0 - - # Loop through the child periods and sum up all the values - for period in self.child_periods_with_data(only_aggregated=True): - try: - period_sum += Decimal(period.actual_value) - except (InvalidOperation, TypeError): - pass - - if include_self: - try: - period_sum += Decimal(self.recalculate_period(save=False, only_self=True)) - except (InvalidOperation, TypeError): - pass - - return str(period_sum) - - def child_periods_percentage(self): - """Returns percentage calculated from the child periods. - - :return String of numerator and denominator - - """ - period_numerator = 0 - period_denominator = 0 - for period in self.child_periods_with_data(only_aggregated=True): - try: - period_numerator += Decimal(period.numerator) - period_denominator += Decimal(period.denominator) - except (InvalidOperation, TypeError): - pass - return str(period_numerator), str(period_denominator) - def adjacent_period(self, next_period=True): """ Returns the next or previous indicator period, if we can find one with a start date, diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index 8a6542003b..fde5963be3 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -101,7 +101,6 @@ def save(self, recalculate=True, *args, **kwargs): # In case the status is approved, recalculate the period if recalculate and self.status == self.STATUS_APPROVED_CODE: # FIXME: Should we call this even when status is not approved? - # self.period.recalculate_period() self.period.update_actual_comment() # Update score even when the update is not approved, yet. It handles the # case where an approved update is returned for revision, etc. @@ -114,7 +113,6 @@ def delete(self, *args, **kwargs): # In case the status was approved, recalculate the period if old_status == self.STATUS_APPROVED_CODE: - # self.period.recalculate_period() self.period.update_actual_comment() self.period.update_score() From 8db72c68b7d15f6131ef6484a7dad84ccdea51b4 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Thu, 20 Oct 2022 10:08:27 +0700 Subject: [PATCH 16/59] [#5140] Implement indicator period jobs runner command --- .../commands/run_aggregation_jobs.py | 9 +++ akvo/rsr/models/result/disaggregation.py | 13 ++++ .../result/disaggregation_aggregation.py | 3 +- .../models/result/indicator_period_data.py | 4 ++ .../results_framework/test_aggregation.py | 65 ++++++++++++++++--- .../tests/usecases/jobs/test_aggregation.py | 17 +++++ akvo/rsr/usecases/jobs/aggregation.py | 11 ++-- .../rsr/usecases/period_update_aggregation.py | 22 +++++-- akvo/settings/90-finish.conf | 1 + 9 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 akvo/rsr/management/commands/run_aggregation_jobs.py diff --git a/akvo/rsr/management/commands/run_aggregation_jobs.py b/akvo/rsr/management/commands/run_aggregation_jobs.py new file mode 100644 index 0000000000..2332060480 --- /dev/null +++ b/akvo/rsr/management/commands/run_aggregation_jobs.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from akvo.rsr.usecases.jobs.aggregation import execute_aggregation_jobs + + +class Command(BaseCommand): + help = "Run indicator period aggregation jobs." + + def handle(self, *args, **options): + execute_aggregation_jobs() diff --git a/akvo/rsr/models/result/disaggregation.py b/akvo/rsr/models/result/disaggregation.py index 30aa7e4d32..d7a37d91cb 100644 --- a/akvo/rsr/models/result/disaggregation.py +++ b/akvo/rsr/models/result/disaggregation.py @@ -9,6 +9,7 @@ from akvo.rsr.fields import ValidXMLTextField from akvo.rsr.mixins import TimestampsMixin, IndicatorUpdateMixin from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE, QUALITATIVE +from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job from django.db import models from django.db.models import signals @@ -69,6 +70,18 @@ def update_incomplete_data(self): or denominator != self.update.denominator) self.siblings().update(incomplete_data=incomplete_data) + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + if self.update.status == IndicatorPeriodData.STATUS_APPROVED_CODE: + schedule_aggregation_job(self.update.period) + + def delete(self, *args, **kwargs): + old_status = self.update.status + period = self.update.period + super().delete(*args, **kwargs) + if old_status == IndicatorPeriodData.STATUS_APPROVED_CODE: + schedule_aggregation_job(period) + @receiver(signals.post_save, sender=Disaggregation) def mark_incomplete_disaggregations(sender, **kwargs): diff --git a/akvo/rsr/models/result/disaggregation_aggregation.py b/akvo/rsr/models/result/disaggregation_aggregation.py index 1fc94ae3fa..0a88c4ed6f 100644 --- a/akvo/rsr/models/result/disaggregation_aggregation.py +++ b/akvo/rsr/models/result/disaggregation_aggregation.py @@ -4,9 +4,9 @@ # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. +from django.apps import apps from django.db.models import Sum from akvo.utils import ensure_decimal -from .indicator_period_data import IndicatorPeriodData class DisaggregationAggregation(object): @@ -33,6 +33,7 @@ def aggregate(self, period, dimension_value): self.aggregate(period.parent_period, dimension_value.parent_dimension_value) def _get_local_values(self, period, dimension_value): + IndicatorPeriodData = apps.get_model('rsr', 'IndicatorPeriodData') return self.disaggregations.filter( update__period=period, update__status=IndicatorPeriodData.STATUS_APPROVED_CODE, diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index fde5963be3..9be226e1b3 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -21,6 +21,7 @@ from akvo.rsr.fields import ValidXMLCharField, ValidXMLTextField from akvo.rsr.mixins import TimestampsMixin, IndicatorUpdateMixin from akvo.utils import rsr_image_path +from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job class IndicatorPeriodData(TimestampsMixin, IndicatorUpdateMixin, models.Model): @@ -101,6 +102,7 @@ def save(self, recalculate=True, *args, **kwargs): # In case the status is approved, recalculate the period if recalculate and self.status == self.STATUS_APPROVED_CODE: # FIXME: Should we call this even when status is not approved? + schedule_aggregation_job(self.period) self.period.update_actual_comment() # Update score even when the update is not approved, yet. It handles the # case where an approved update is returned for revision, etc. @@ -108,11 +110,13 @@ def save(self, recalculate=True, *args, **kwargs): def delete(self, *args, **kwargs): old_status = self.status + period = self.period super(IndicatorPeriodData, self).delete(*args, **kwargs) # In case the status was approved, recalculate the period if old_status == self.STATUS_APPROVED_CODE: + schedule_aggregation_job(period) self.period.update_actual_comment() self.period.update_score() diff --git a/akvo/rsr/tests/results_framework/test_aggregation.py b/akvo/rsr/tests/results_framework/test_aggregation.py index fc8b6a21f4..4e9ff3ce2e 100644 --- a/akvo/rsr/tests/results_framework/test_aggregation.py +++ b/akvo/rsr/tests/results_framework/test_aggregation.py @@ -12,16 +12,17 @@ import unittest from akvo.rsr.models import (Project, Result, Indicator, IndicatorPeriod, - IndicatorPeriodData, User, RelatedProject) + IndicatorPeriodData, User, RelatedProject, + Disaggregation, IndicatorDimensionName, IndicatorDimensionValue) from akvo.rsr.models.result.utils import (calculate_percentage, MultipleUpdateError) from akvo.rsr.usecases.period_update_aggregation import aggregate +from akvo.rsr.usecases.jobs import aggregation as jobs from django.test import TestCase -class UnitAggregationTestCase(TestCase): - +class AggregationTestCase(TestCase): indicator_measure = '1' APPROVED = IndicatorPeriodData.STATUS_APPROVED_CODE @@ -50,9 +51,8 @@ def setUp(self): period_end=datetime.date.today() + datetime.timedelta(days=1), target_value="100" ) - - # Import status - self.import_status = 0 + self.dimension_name = IndicatorDimensionName.objects.create(project=self.parent_project) + self.dimension_value = IndicatorDimensionValue.objects.create(name=self.dimension_name) @staticmethod def create_published_project(title): @@ -74,12 +74,12 @@ def create_child_project(self, title): ) return child_project - def create_indicator_period_update(self, value, indicator_period=None): + def create_indicator_period_update(self, value, indicator_period=None, status=IndicatorPeriodData.STATUS_APPROVED_CODE): indicator_period_data = IndicatorPeriodData.objects.create( period=indicator_period if indicator_period is not None else self.period, user=self.user, value=value, - status=self.APPROVED + status=status ) return indicator_period_data @@ -88,6 +88,9 @@ def get_child_period(self, indicator_period, child_project=None): child_project = self.child_project return indicator_period.child_periods.get(indicator__result__project=child_project) + +class UnitAggregationTestCase(AggregationTestCase): + def test_should_set_period_actual_value(self): # Given actual_value = '40' @@ -277,8 +280,52 @@ def test_should_not_aggregate_excluded_child_period_values(self): period = IndicatorPeriod.objects.get(id=self.period.id) self.assertEqual(Decimal(value), Decimal(period.actual_value)) + def test_schedule_aggregation_job_on_indicator_update_object(self): + self.assertEqual(0, jobs.get_scheduled_jobs().count(), 'No scheduled jobs initially') + + update = self.create_indicator_period_update(value=0, status=IndicatorPeriodData.STATUS_PENDING_CODE) + + self.assertEqual(0, jobs.get_scheduled_jobs().count(), 'No scheduled jobs before update approved') + + update.status = IndicatorPeriodData.STATUS_APPROVED_CODE + update.save() + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'A job scheduled after update gets approved') + + jobs.get_scheduled_jobs().first().mark_running() + + self.assertEqual(0, jobs.get_scheduled_jobs().count(), 'No scheduled jobs when the job is running') + + update.delete() + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'A job scheduled after update gets deleted') + + def test_schedule_aggregation_job_on_disaggregation_object(self): + self.assertEqual(0, jobs.get_scheduled_jobs().count(), 'No scheduled jobs initially') + + update = self.create_indicator_period_update(value=0) + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'A job scheduled on approved update') + + disaggregation = Disaggregation.objects.create(update=update, dimension_value=self.dimension_value) + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'No additional job scheduled if already exists for the same period') + + jobs.get_scheduled_jobs().first().mark_running() + + disaggregation.value = 1 + disaggregation.save() + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'A job scheduled on disaggregation modification') + + jobs.get_scheduled_jobs().first().mark_running() + + disaggregation.delete() + + self.assertEqual(1, jobs.get_scheduled_jobs().count(), 'A job scheduled after disaggregation gets deleted') + -class PercentageAggregationTestCase(UnitAggregationTestCase): +class PercentageAggregationTestCase(AggregationTestCase): """Tests the aggregation of percentage measure indicators.""" indicator_measure = '2' diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 127b7d515d..dea1460547 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -1,5 +1,6 @@ import datetime +from unittest.mock import patch, MagicMock from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob from akvo.rsr.tests.base import BaseTestCase @@ -140,3 +141,19 @@ def test_with_live_job(self): dead_jobs = usecases.fail_dead_jobs() self.assertListEqual(dead_jobs, []) + + +class AggregationJobRunnerTestCase(AggregationJobBaseTests): + + def test_finished_jobs(self): + usecases.execute_aggregation_jobs() + self.assertEqual(0, usecases.get_scheduled_jobs().count()) + self.assertEqual(1, usecases.get_finished_jobs().count()) + + @patch('akvo.rsr.usecases.jobs.aggregation.email_failed_job_owner') + def test_failed_jobs(self, *_): + with patch('akvo.rsr.usecases.jobs.aggregation.run_aggregation', new=MagicMock(side_effect=Exception('Fail job'))): + usecases.execute_aggregation_jobs() + self.assertEqual(0, usecases.get_scheduled_jobs().count()) + self.assertEqual(0, usecases.get_finished_jobs().count()) + self.assertEqual(1, usecases.get_failed_jobs().count()) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index e713841266..155c5616b8 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -1,10 +1,14 @@ +from __future__ import annotations from django.db.models import QuerySet from django.db.transaction import atomic -from typing import List +from typing import List, TYPE_CHECKING + +if TYPE_CHECKING: + from akvo.rsr.models import IndicatorPeriod -from akvo.rsr.models import IndicatorPeriod from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob from akvo.rsr.models.cron_job import CronJobMixin +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.usecases.jobs.cron import is_job_dead @@ -100,5 +104,4 @@ def email_failed_job_owner(failed_job: IndicatorPeriodAggregationJob, reason: st def run_aggregation(period: IndicatorPeriod): - # TODO: Implement - raise NotImplementedError() + aggregate(period) diff --git a/akvo/rsr/usecases/period_update_aggregation.py b/akvo/rsr/usecases/period_update_aggregation.py index 3477fb0645..0fdabba8db 100644 --- a/akvo/rsr/usecases/period_update_aggregation.py +++ b/akvo/rsr/usecases/period_update_aggregation.py @@ -1,14 +1,22 @@ +from __future__ import annotations from decimal import Decimal -from typing import Tuple, Optional +from typing import Tuple, Optional, TYPE_CHECKING +from django.apps import apps from django.db import transaction from django.db.models import QuerySet, Q, Sum from akvo.utils import ensure_decimal -from akvo.rsr.models import IndicatorPeriod, IndicatorPeriodData, Disaggregation, IndicatorDimensionValue + +if TYPE_CHECKING: + from akvo.rsr.models import IndicatorPeriod + from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE, calculate_percentage -from akvo.rsr.models.result.indicator_period_disaggregation import IndicatorPeriodDisaggregation from akvo.rsr.models.result.disaggregation_aggregation import DisaggregationAggregation -disaggregation_aggregation = DisaggregationAggregation(Disaggregation.objects, IndicatorPeriodDisaggregation.objects) + +def get_disaggregation_aggregation(): + Disaggregation = apps.get_model('rsr', 'Disaggregation') + IndicatorPeriodDisaggregation = apps.get_model('rsr', 'IndicatorPeriodDisaggregation') + return DisaggregationAggregation(Disaggregation.objects, IndicatorPeriodDisaggregation.objects) @transaction.atomic @@ -38,13 +46,17 @@ def _aggregate_period_value(period: IndicatorPeriod): def _aggregate_disaggregation(period: IndicatorPeriod): + Disaggregation = apps.get_model('rsr', 'Disaggregation') + IndicatorPeriodData = apps.get_model('rsr', 'IndicatorPeriodData') + IndicatorDimensionValue = apps.get_model('rsr', 'IndicatorDimensionValue') + disaggregations = Disaggregation.objects.filter(update__period=period, update__status=IndicatorPeriodData.STATUS_APPROVED_CODE) dimension_values = ( IndicatorDimensionValue.objects.filter(name__in=period.indicator.dimension_names.all()) | IndicatorDimensionValue.objects.filter(disaggregations__in=disaggregations) ).distinct() for dimension_value in dimension_values: - disaggregation_aggregation.aggregate(period, dimension_value) + get_disaggregation_aggregation().aggregate(period, dimension_value) def sum_updates(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]: diff --git a/akvo/settings/90-finish.conf b/akvo/settings/90-finish.conf index e7ff0c2e06..6e89fd7967 100644 --- a/akvo/settings/90-finish.conf +++ b/akvo/settings/90-finish.conf @@ -104,6 +104,7 @@ CRONTAB_COMMAND_SUFFIX = '> /proc/1/fd/1 2>/proc/1/fd/2' CRONJOBS = [ ('* * * * *', 'django.core.management.call_command', ['iati_export']), ('* * * * *', 'django.core.management.call_command', ['send_report_via_email']), + ('* * * * *', 'django.core.management.call_command', ['run_aggregation_jobs']), ('10 2 * * *', 'django.core.management.call_command', ['a4a_optimy_import']), ('*/5 * * * *', 'django.core.management.call_command', ['perform_iati_checks']), ('0 0 * * *', 'django.core.management.call_command', ['cleanup_untitled_and_unpublished_projects']), From 3760a05af155f106ee368de9f064804ca938f047 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Mon, 24 Oct 2022 10:32:56 +0700 Subject: [PATCH 17/59] [#5075] Fix project editor nothing was returned from render. After reviewing each component in section5.jsx, I found the issue came from `editor/section5/indicators.jsx` file. Which is `allowIndicatorLabels` returning false so that component doesn't return from render. so I highly recommend in the future not to put logic inside component ``. [#5073] Fix long delay in desc due to endpoint issue --- .../spa/app/modules/editor/action-types.js | 3 ++- akvo/rsr/spa/app/modules/editor/actions.js | 1 + .../modules/editor/project-init-handler.js | 23 +++++++++++++++---- akvo/rsr/spa/app/modules/editor/reducer.js | 11 +++++++++ .../modules/editor/section5/indicators.jsx | 8 ++++--- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/akvo/rsr/spa/app/modules/editor/action-types.js b/akvo/rsr/spa/app/modules/editor/action-types.js index 5c2d1a1cbb..a6fbbf73c3 100644 --- a/akvo/rsr/spa/app/modules/editor/action-types.js +++ b/akvo/rsr/spa/app/modules/editor/action-types.js @@ -24,5 +24,6 @@ export default { UPDATE_LAST_SAVED: 'PE_UPDATE_LAST_SAVED', SET_FIELD_REQUIRED_ERROR: 'PE_SET_FIELD_REQUIRED_ERROR', UPDATE_PAGINATION: 'PE_UPDATE_PAGINATION', - VALIDATION_SYNC: 'PE_VALIDATION_SYNC' + VALIDATION_SYNC: 'PE_VALIDATION_SYNC', + SET_FIRST_SECTION_ID: 'PE_SET_FIRST_SECTION_ID', } diff --git a/akvo/rsr/spa/app/modules/editor/actions.js b/akvo/rsr/spa/app/modules/editor/actions.js index d2a938a74a..44ef5add81 100644 --- a/akvo/rsr/spa/app/modules/editor/actions.js +++ b/akvo/rsr/spa/app/modules/editor/actions.js @@ -96,3 +96,4 @@ export const setProjectStatus = (publishingStatus, hasHierarchy, needsReportingT dispatch({ type: actionTypes.SAVE_FIELDS, fields: { publishingStatus, hasHierarchy, needsReportingTimeoutDays, pendingUpdateCount, canEditProject }, sectionIndex: 1, noSync: true }) } export const setUser = (user) => ({ type: 'SET_USER', user }) +export const setFirstSectionID = projectId => ({ type: actionTypes.SET_FIRST_SECTION_ID, projectId }) diff --git a/akvo/rsr/spa/app/modules/editor/project-init-handler.js b/akvo/rsr/spa/app/modules/editor/project-init-handler.js index b234e96a26..4065eccf01 100644 --- a/akvo/rsr/spa/app/modules/editor/project-init-handler.js +++ b/akvo/rsr/spa/app/modules/editor/project-init-handler.js @@ -1,5 +1,5 @@ /* eslint-disable no-restricted-globals */ -import React, { useEffect, useState } from 'react' +import { useEffect, useState } from 'react' import { connect } from 'react-redux' import api from '../../utils/api' import { endpoints, getTransform } from './endpoints' @@ -54,6 +54,13 @@ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { props.fetchFields(SECTION_INFO, data) props.setSectionFetched(SECTION_INFO) }) + .catch(() => { + /** + * in order to stop the loading indicator, + * we need to set the ID by sending the params id + */ + props.setFirstSectionID(params.id) + }) const fetchNextSection = index => { const _endpoints = endpoints[`section${index}`] || endpoints.section1 @@ -71,7 +78,9 @@ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { .get(endpoint, _params, getTransform(index, setName, 'response')) .then(({ data: { results, count } }) => { props.fetchSetItems(index, setName, results, count) - props.setSectionFetched(index) + setTimeout(() => { + props.setSectionFetched(index) + }, 1000) }) }) } @@ -84,7 +93,10 @@ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { setPreload(false) if (params.id !== editorRdr.projectId || !editorRdr.section1.isFetched) { /** - * if previous ID not equal with current ID then + * if the previous ID is not the same as the current ID + * or + * section1 hasn't been fetched + * then * Reset all states and set old ID with current ID */ props.resetProject() @@ -109,8 +121,9 @@ const ProjectInitHandler = ({ match: { params }, editorRdr, ...props }) => { setNextSectionIndex(next) } /** - * Once section1 is fetched then - * Update each section that has a dependency with section1 + * ============================================ + * editorRdr?.section1?.fields?.id is a blocker + * to ensure all required fields are available */ if (editorRdr?.section1?.fields?.id) { sectionInstanceToRoot.forEach((index) => { diff --git a/akvo/rsr/spa/app/modules/editor/reducer.js b/akvo/rsr/spa/app/modules/editor/reducer.js index 53546cd059..f785463710 100644 --- a/akvo/rsr/spa/app/modules/editor/reducer.js +++ b/akvo/rsr/spa/app/modules/editor/reducer.js @@ -281,6 +281,17 @@ export default (state = initialState, action) => { case actionTypes.VALIDATION_SYNC: newState[sectionKey].errors = validateSection(sectionKey, state.validations, newState[sectionKey].fields) return newState + case actionTypes.SET_FIRST_SECTION_ID: + return { + ...state, + section1: { + ...state.section1, + fields: { + ...state.section1.fields, + id: action.projectId + } + }, + } default: return state } } diff --git a/akvo/rsr/spa/app/modules/editor/section5/indicators.jsx b/akvo/rsr/spa/app/modules/editor/section5/indicators.jsx index b26d24ebd3..aef8c61d52 100644 --- a/akvo/rsr/spa/app/modules/editor/section5/indicators.jsx +++ b/akvo/rsr/spa/app/modules/editor/section5/indicators.jsx @@ -263,9 +263,11 @@ const Indicators = connect(null, { addSetItem, removeSetItem, moveSetItem })( /> - - {allowIndicatorLabels && } - + {allowIndicatorLabels && ( + + + + )}
{!isImported(index) && From 317d6a3ed93ae689685f4ae4bdec587519201922 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 24 Oct 2022 11:56:35 +0200 Subject: [PATCH 18/59] chore: add TODO for email_failed_job_owner For testing, it shouldn't throw an exception #5140: Stabilize period update aggregations --- akvo/rsr/usecases/jobs/aggregation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 155c5616b8..cd7228fe01 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -100,7 +100,8 @@ def fail_dead_jobs() -> List[IndicatorPeriodAggregationJob]: def email_failed_job_owner(failed_job: IndicatorPeriodAggregationJob, reason: str): - raise NotImplementedError() + # TODO: implement + pass def run_aggregation(period: IndicatorPeriod): From 73e7c243c693998d0c617a37ca457582fbe237e4 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 24 Oct 2022 16:45:42 +0200 Subject: [PATCH 19/59] feat: Send failed aggregation job email #5140: Stabilize period update aggregations --- .../0221_indicatorperiodaggregationjob.py | 7 ++++ akvo/rsr/models/employment.py | 5 +++ .../tests/usecases/jobs/test_aggregation.py | 31 ++++++++++++++-- akvo/rsr/usecases/jobs/aggregation.py | 35 ++++++++++++++----- .../indicator_aggregation/fail_message.html | 31 ++++++++++++++++ .../indicator_aggregation/fail_subject.txt | 1 + 6 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 akvo/templates/indicator_aggregation/fail_message.html create mode 100644 akvo/templates/indicator_aggregation/fail_subject.txt diff --git a/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py index a3354bbb05..1e04cc1053 100644 --- a/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py +++ b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py @@ -26,4 +26,11 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.AddField( + model_name='employment', + name='receives_indicator_aggregation_emails', + field=models.BooleanField(default=False, + help_text='Some events of indicator aggregations in projects of this org will trigger emails', + verbose_name='Receive indicator emails'), + ), ] diff --git a/akvo/rsr/models/employment.py b/akvo/rsr/models/employment.py index d0ebd6bb1f..4e95581b29 100644 --- a/akvo/rsr/models/employment.py +++ b/akvo/rsr/models/employment.py @@ -40,6 +40,11 @@ class Employment(models.Model): is_approved = models.BooleanField(_('approved'), default=False, help_text=_('Designates whether this employment is approved ' 'by an administrator.')) + receives_indicator_aggregation_emails = models.BooleanField( + _('Receive indicator emails'), + default=False, + help_text=_('Some events of indicator aggregations in projects of this org will trigger emails'), + ) country = ValidXMLCharField( _('country'), blank=True, max_length=2, choices=codelist_choices(COUNTRY, show_code=False) ) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index dea1460547..0ec2df5438 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -1,8 +1,12 @@ import datetime from unittest.mock import patch, MagicMock + +from django.core import mail + from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob +from akvo.rsr.permissions import GROUP_NAME_ME_MANAGERS from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.usecases.jobs import aggregation as usecases from akvo.rsr.usecases.jobs.cron import is_job_dead @@ -19,6 +23,8 @@ def setUp(self): ) self.project = self.create_project('Test project') + self.org = self.create_organisation("Test org") + self.project.set_reporting_org(self.org) # Create results framework self.result = Result.objects.create( @@ -145,15 +151,34 @@ def test_with_live_job(self): class AggregationJobRunnerTestCase(AggregationJobBaseTests): + def setUp(self): + super().setUp() + mail.outbox = [] + def test_finished_jobs(self): usecases.execute_aggregation_jobs() self.assertEqual(0, usecases.get_scheduled_jobs().count()) self.assertEqual(1, usecases.get_finished_jobs().count()) - @patch('akvo.rsr.usecases.jobs.aggregation.email_failed_job_owner') - def test_failed_jobs(self, *_): - with patch('akvo.rsr.usecases.jobs.aggregation.run_aggregation', new=MagicMock(side_effect=Exception('Fail job'))): + def test_failed_jobs(self): + # Employ to receive failed job email + employment = self.make_employment(self.user, self.org, GROUP_NAME_ME_MANAGERS) + employment.receives_indicator_aggregation_emails = True + employment.save() + + # Employ user who won't receive the failed job email + self.make_employment(self.create_user("another_user@doing.test"), self.org, GROUP_NAME_ME_MANAGERS) + mail.outbox = [] + + with patch('akvo.rsr.usecases.jobs.aggregation.run_aggregation', + new=MagicMock(side_effect=Exception('Fail job'))): usecases.execute_aggregation_jobs() self.assertEqual(0, usecases.get_scheduled_jobs().count()) self.assertEqual(0, usecases.get_finished_jobs().count()) self.assertEqual(1, usecases.get_failed_jobs().count()) + + # Ensure the failed job email was sent out + msg = mail.outbox[0] + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(msg.to, [self.user.email]) + self.assertEqual(msg.subject, 'An indicator aggregation job failed') diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index cd7228fe01..8b8540f706 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -1,7 +1,12 @@ from __future__ import annotations + +import logging +from typing import List, TYPE_CHECKING + from django.db.models import QuerySet from django.db.transaction import atomic -from typing import List, TYPE_CHECKING + +from akvo.utils import rsr_send_mail_to_users if TYPE_CHECKING: from akvo.rsr.models import IndicatorPeriod @@ -11,6 +16,8 @@ from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.usecases.jobs.cron import is_job_dead +logger = logging.getLogger(__name__) + def get_scheduled_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: return IndicatorPeriodAggregationJob.objects.filter( @@ -67,9 +74,12 @@ def execute_aggregation_jobs(): try: run_aggregation(scheduled_job.period) scheduled_job.mark_finished() - except Exception as e: + except: scheduled_job.mark_failed() - email_failed_job_owner(scheduled_job, str(e)) + # Only email them the first time + if scheduled_job.attempts <= 1: + email_failed_job_owners(scheduled_job) + logger.error("Failed executing aggregation job %s", scheduled_job.id) @atomic @@ -79,8 +89,7 @@ def handle_failed_jobs(): failed_jobs = get_failed_jobs() for failed_job in failed_jobs: - email_failed_job_owner(failed_job, "Job died") - + email_failed_job_owners(failed_job) failed_job.mark_scheduled() @@ -99,9 +108,19 @@ def fail_dead_jobs() -> List[IndicatorPeriodAggregationJob]: return dead_jobs -def email_failed_job_owner(failed_job: IndicatorPeriodAggregationJob, reason: str): - # TODO: implement - pass +def email_failed_job_owners(failed_job: IndicatorPeriodAggregationJob): + recipients = failed_job.program.primary_organisation.employees.filter( + receives_indicator_aggregation_emails=True + ).select_related("user") + rsr_send_mail_to_users( + [recipient.user for recipient in recipients], + subject="indicator_aggregation/fail_subject.txt", + message="indicator_aggregation/fail_message.html", + msg_context={ + "indicator": failed_job.period.indicator, + "program": failed_job.program, + } + ) def run_aggregation(period: IndicatorPeriod): diff --git a/akvo/templates/indicator_aggregation/fail_message.html b/akvo/templates/indicator_aggregation/fail_message.html new file mode 100644 index 0000000000..2f8f81d440 --- /dev/null +++ b/akvo/templates/indicator_aggregation/fail_message.html @@ -0,0 +1,31 @@ + + +{% load i18n %} + + + + + + + {% blocktrans %} +

The indicator: "{{ indicator.title }}" in the hierarchy of the program + {{ program.title }} failed. +

+ +

The job failed is scheduled to rerun shortly.

+

You shall receive an email once the indicator has been successfully updated

+ {% endblocktrans %} + + + diff --git a/akvo/templates/indicator_aggregation/fail_subject.txt b/akvo/templates/indicator_aggregation/fail_subject.txt new file mode 100644 index 0000000000..f1f3b6f297 --- /dev/null +++ b/akvo/templates/indicator_aggregation/fail_subject.txt @@ -0,0 +1 @@ +{% load i18n %}{% trans 'An indicator aggregation job failed' %} \ No newline at end of file From eae5f8e95fa54b3e48312237d91026ff26285894 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 25 Oct 2022 09:45:22 +0200 Subject: [PATCH 20/59] feat: Send success aggregation job email When a previously failed job succeeds, it sends a notification to users of the organisation that should receive them. Some methods were also refactored to make code more legible (get_job_recipients()) and reduce the number of requests (get_base_jobs()) #5140: Stabilize period update aggregations --- .../0221_indicatorperiodaggregationjob.py | 4 +- akvo/rsr/models/aggregation_job.py | 6 +- .../tests/usecases/jobs/test_aggregation.py | 37 ++++++-- akvo/rsr/usecases/jobs/aggregation.py | 90 ++++++++++++------- .../indicator_aggregation/fail_message.html | 16 ++-- .../success_message.html | 26 ++++++ .../indicator_aggregation/success_subject.txt | 1 + 7 files changed, 128 insertions(+), 52 deletions(-) create mode 100644 akvo/templates/indicator_aggregation/success_message.html create mode 100644 akvo/templates/indicator_aggregation/success_subject.txt diff --git a/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py index 1e04cc1053..7435e31a04 100644 --- a/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py +++ b/akvo/rsr/migrations/0221_indicatorperiodaggregationjob.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.10 on 2022-10-18 07:03 +# Generated by Django 3.2.10 on 2022-10-25 07:53 from django.db import migrations, models import django.db.models.deletion @@ -19,7 +19,7 @@ class Migration(migrations.Migration): ('attempts', models.IntegerField(default=0)), ('pid', models.PositiveIntegerField(null=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rsr.indicatorperiod')), + ('period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='aggregation_jobs', to='rsr.indicatorperiod')), ('program', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rsr.project')), ], options={ diff --git a/akvo/rsr/models/aggregation_job.py b/akvo/rsr/models/aggregation_job.py index a00a90c4aa..19419eed31 100644 --- a/akvo/rsr/models/aggregation_job.py +++ b/akvo/rsr/models/aggregation_job.py @@ -4,5 +4,9 @@ class IndicatorPeriodAggregationJob(CronJobMixin): - period = models.ForeignKey("IndicatorPeriod", on_delete=models.CASCADE) + period = models.ForeignKey( + "IndicatorPeriod", + on_delete=models.CASCADE, + related_name="aggregation_jobs" + ) program = models.ForeignKey("Project", on_delete=models.CASCADE) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 0ec2df5438..352757bba1 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -153,14 +153,7 @@ class AggregationJobRunnerTestCase(AggregationJobBaseTests): def setUp(self): super().setUp() - mail.outbox = [] - def test_finished_jobs(self): - usecases.execute_aggregation_jobs() - self.assertEqual(0, usecases.get_scheduled_jobs().count()) - self.assertEqual(1, usecases.get_finished_jobs().count()) - - def test_failed_jobs(self): # Employ to receive failed job email employment = self.make_employment(self.user, self.org, GROUP_NAME_ME_MANAGERS) employment.receives_indicator_aggregation_emails = True @@ -168,8 +161,19 @@ def test_failed_jobs(self): # Employ user who won't receive the failed job email self.make_employment(self.create_user("another_user@doing.test"), self.org, GROUP_NAME_ME_MANAGERS) + + # Employment creation sends out emails that we don't care about mail.outbox = [] + def test_finished_jobs(self): + usecases.execute_aggregation_jobs() + self.assertEqual(0, usecases.get_scheduled_jobs().count()) + self.assertEqual(1, usecases.get_finished_jobs().count()) + + # Make sure no success email was sent out + self.assertEqual(len(mail.outbox), 0) + + def test_failed_jobs(self): with patch('akvo.rsr.usecases.jobs.aggregation.run_aggregation', new=MagicMock(side_effect=Exception('Fail job'))): usecases.execute_aggregation_jobs() @@ -178,7 +182,22 @@ def test_failed_jobs(self): self.assertEqual(1, usecases.get_failed_jobs().count()) # Ensure the failed job email was sent out - msg = mail.outbox[0] self.assertEqual(len(mail.outbox), 1) + msg = mail.outbox[0] + self.assertEqual(msg.to, [self.user.email]) + self.assertEqual(msg.subject, "An indicator aggregation job failed") + + def test_success_after_failure(self): + """A job that succeeds after previously failing should send out a success email""" + with patch('akvo.rsr.usecases.jobs.aggregation.run_aggregation', + new=MagicMock(side_effect=Exception('Fail job'))): + usecases.execute_aggregation_jobs() + + # Rerun successfully + usecases.execute_aggregation_jobs() + + # Ensure the success job email was sent out + self.assertEqual(len(mail.outbox), 2) + msg = mail.outbox[1] self.assertEqual(msg.to, [self.user.email]) - self.assertEqual(msg.subject, 'An indicator aggregation job failed') + self.assertEqual(msg.subject, "Previously failed indicator aggregation job has succeeded") diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 8b8540f706..fd838665e5 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -20,33 +20,27 @@ def get_scheduled_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: - return IndicatorPeriodAggregationJob.objects.filter( - status=CronJobMixin.Status.SCHEDULED, - ) + return base_get_jobs().filter(status=CronJobMixin.Status.SCHEDULED) def get_running_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: - return IndicatorPeriodAggregationJob.objects.filter( - status=CronJobMixin.Status.RUNNING, - ) + return base_get_jobs().filter(status=CronJobMixin.Status.RUNNING) def get_failed_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: - return IndicatorPeriodAggregationJob.objects.filter( - status=CronJobMixin.Status.FAILED, - ) + return base_get_jobs().filter(status=CronJobMixin.Status.FAILED) def get_maxxed_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: - return IndicatorPeriodAggregationJob.objects.filter( - status=CronJobMixin.Status.MAXXED, - ) + return base_get_jobs().filter(status=CronJobMixin.Status.MAXXED) def get_finished_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: - return IndicatorPeriodAggregationJob.objects.filter( - status=CronJobMixin.Status.FINISHED, - ) + return base_get_jobs().filter(status=CronJobMixin.Status.FINISHED) + + +def base_get_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: + return IndicatorPeriodAggregationJob.objects.select_related("period__indicator") def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorPeriodAggregationJob: @@ -74,22 +68,36 @@ def execute_aggregation_jobs(): try: run_aggregation(scheduled_job.period) scheduled_job.mark_finished() - except: + except Exception as e: scheduled_job.mark_failed() - # Only email them the first time + # Only send job failure email the first time if scheduled_job.attempts <= 1: - email_failed_job_owners(scheduled_job) - logger.error("Failed executing aggregation job %s", scheduled_job.id) + email_job_owners( + scheduled_job, + "indicator_aggregation/fail_subject.txt", + "indicator_aggregation/fail_message.html", + reason=str(e) + ) + logger.error("Failed executing aggregation job %s: %s", scheduled_job.id, e) + else: + # Send out success email if job previously failed + if scheduled_job.attempts > 1: + email_job_owners( + scheduled_job, + "indicator_aggregation/success_subject.txt", + "indicator_aggregation/success_message.html", + ) + + +def run_aggregation(period: IndicatorPeriod): + aggregate(period) @atomic def handle_failed_jobs(): - """Identify failed jobs, notify owners, and reschedule them""" + """Identify failed jobs and reschedule them""" fail_dead_jobs() - failed_jobs = get_failed_jobs() - - for failed_job in failed_jobs: - email_failed_job_owners(failed_job) + for failed_job in get_failed_jobs(): failed_job.mark_scheduled() @@ -105,23 +113,41 @@ def fail_dead_jobs() -> List[IndicatorPeriodAggregationJob]: running_job.mark_failed() dead_jobs.append(running_job) + if running_job.attempts <= 1: + email_job_owners( + running_job, + "indicator_aggregation/fail_subject.txt", + "indicator_aggregation/fail_message.html", + reason="Process died", + ) + logger.warning( + "Aggregation job died. ID %s. Indicator '%s'", + running_job.id, + running_job.period.indicator.title, + ) + return dead_jobs -def email_failed_job_owners(failed_job: IndicatorPeriodAggregationJob): - recipients = failed_job.program.primary_organisation.employees.filter( - receives_indicator_aggregation_emails=True - ).select_related("user") +def email_job_owners( + failed_job: IndicatorPeriodAggregationJob, + subject_template: str, message_template: str, + reason: str = None +): + recipients = get_job_recipients(failed_job) rsr_send_mail_to_users( [recipient.user for recipient in recipients], - subject="indicator_aggregation/fail_subject.txt", - message="indicator_aggregation/fail_message.html", + subject=subject_template, + message=message_template, msg_context={ "indicator": failed_job.period.indicator, "program": failed_job.program, + "reason": reason, } ) -def run_aggregation(period: IndicatorPeriod): - aggregate(period) +def get_job_recipients(job: IndicatorPeriodAggregationJob): + return job.program.primary_organisation.employees.filter( + receives_indicator_aggregation_emails=True + ).select_related("user") diff --git a/akvo/templates/indicator_aggregation/fail_message.html b/akvo/templates/indicator_aggregation/fail_message.html index 2f8f81d440..7d5111df30 100644 --- a/akvo/templates/indicator_aggregation/fail_message.html +++ b/akvo/templates/indicator_aggregation/fail_message.html @@ -10,20 +10,20 @@ line-height: 1.42857; color: #394c50; } - h1, h3 { - font-family: 'Montserrat', 'Helvetica Neue', Helvetica, Arial, sans-serif; - font-weight: normal; - color: #2C2A74; - } {% blocktrans %} -

The indicator: "{{ indicator.title }}" in the hierarchy of the program - {{ program.title }} failed. +

The indicator "{{ indicator.title }}" in the hierarchy of the program + {{ program.title }} + had triggered an aggregation job.

-

The job failed is scheduled to rerun shortly.

+

Reason: {{ reason }}

+ +

The job failed and couldn't complete aggregation.

+

It shall be rerun shortly.

+

You shall receive an email once the indicator has been successfully updated

{% endblocktrans %} diff --git a/akvo/templates/indicator_aggregation/success_message.html b/akvo/templates/indicator_aggregation/success_message.html new file mode 100644 index 0000000000..01d3c28950 --- /dev/null +++ b/akvo/templates/indicator_aggregation/success_message.html @@ -0,0 +1,26 @@ + + +{% load i18n %} + + + + + + + {% blocktrans %} +

The indicator: "{{ indicator.title }}" in the hierarchy of the program + {{ program.title }} + had triggered a job that had failed. +

+ +

After rerunning the job, it has now succeeded and the aggregation is complete.

+ {% endblocktrans %} + + + diff --git a/akvo/templates/indicator_aggregation/success_subject.txt b/akvo/templates/indicator_aggregation/success_subject.txt new file mode 100644 index 0000000000..358a2441a3 --- /dev/null +++ b/akvo/templates/indicator_aggregation/success_subject.txt @@ -0,0 +1 @@ +{% load i18n %}{% trans 'Previously failed indicator aggregation job has succeeded' %} \ No newline at end of file From 87cc03bbd7f78be061905d7154d8e02718b1eaa1 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 25 Oct 2022 10:40:31 +0200 Subject: [PATCH 21/59] feat: Log when an aggregation job is created for scheduling #5140: Stabilize period update aggregations --- akvo/rsr/usecases/jobs/aggregation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index fd838665e5..c5c142c8c7 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -47,6 +47,7 @@ def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorPeriodAggregat """ Schedule a job for the period to be aggregated upwards if no job exists """ + logger.info("Scheduling indicator aggregation job for %s: %s", period, period.indicator.title) if existing_job := get_scheduled_jobs().filter(period=period).first(): existing_job.save() return existing_job From a2b9e6a97c0721a7b66203202140bf4d249693a3 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 25 Oct 2022 10:40:46 +0200 Subject: [PATCH 22/59] feat: Add admin page for IndicatorPeriodAggregationJob #5140: Stabilize period update aggregations --- akvo/rsr/admin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/akvo/rsr/admin.py b/akvo/rsr/admin.py index 037ccf7ad9..e1ee6b7f02 100644 --- a/akvo/rsr/admin.py +++ b/akvo/rsr/admin.py @@ -944,6 +944,34 @@ def get_queryset(self, request): admin.site.register(apps.get_model('rsr', 'IndicatorPeriodData'), IndicatorPeriodDataAdmin) +class IndicatorPeriodAggregationJobAdmin(admin.ModelAdmin): + model = apps.get_model('rsr', 'IndicatorPeriodAggregationJob') + list_display = ('status', 'program', 'period', 'indicator_title') + list_filter = ('status', ) + search_fields = ('program__title', 'period__indicator__title') + readonly_fields = ('updated_at', 'period', 'program', 'indicator_title') + + @admin.display(description='Program Title') + def program_title(self, obj): + return obj.program.title + + @admin.display(description='Indicator Title') + def indicator_title(self, obj): + return obj.period.indicator.title + + def get_queryset(self, request): + queryset = super().get_queryset(request).select_related("period__indicator") + if request.user.is_admin or request.user.is_superuser: + return queryset + + employments = request.user.approved_employments(['Admins', 'M&E Managers']) + projects = employments.organisations().all_projects() + return self.model.objects.filter(period__indicator__result__project__in=projects) + + +admin.site.register(apps.get_model('rsr', 'IndicatorPeriodAggregationJob'), IndicatorPeriodAggregationJobAdmin) + + class ReportAdminForm(forms.ModelForm): class Meta: model = apps.get_model('rsr', 'Report') From e6eb53ffd0eae49b5b81188996420cbf10b4c05c Mon Sep 17 00:00:00 2001 From: zuhdil Date: Sat, 29 Oct 2022 00:34:15 +0700 Subject: [PATCH 23/59] [#5140] Lock crontab jobs and make command for program aggregation --- .../recalculate_program_aggregation.py | 30 +++++++++++ akvo/rsr/usecases/jobs/aggregation.py | 50 +++++++++---------- akvo/settings/90-finish.conf | 1 + 3 files changed, 55 insertions(+), 26 deletions(-) create mode 100644 akvo/rsr/management/commands/recalculate_program_aggregation.py diff --git a/akvo/rsr/management/commands/recalculate_program_aggregation.py b/akvo/rsr/management/commands/recalculate_program_aggregation.py new file mode 100644 index 0000000000..204da90037 --- /dev/null +++ b/akvo/rsr/management/commands/recalculate_program_aggregation.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand +from django.db.models import Count, Q + +from akvo.rsr.models import ProjectHierarchy, IndicatorPeriod, IndicatorPeriodData +from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job + + +class Command(BaseCommand): + help = 'Script for recalculating periods aggregation of a program' + + def add_arguments(self, parser): + parser.add_argument('program_id', type=int) + + def handle(self, *args, **options): + try: + hierarchy = ProjectHierarchy.objects.get(root_project=options['program_id']) + program = hierarchy.root_project + except ProjectHierarchy.DoesNotExist: + print("Program not found") + return + + descendants = program.descendants() + periods = IndicatorPeriod.objects\ + .annotate(approved_count=Count('data', filter=Q(data__status=IndicatorPeriodData.STATUS_APPROVED_CODE)))\ + .filter(approved_count__gte=1, indicator__result__project__in=descendants) + + for period in periods: + schedule_aggregation_job(period) + + print(f"Scheduled period aggregation jobs: {periods.count()}, on program: {program.title}") diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index c5c142c8c7..e3b1387e4e 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -62,32 +62,30 @@ def execute_aggregation_jobs(): """ handle_failed_jobs() - scheduled_jobs = get_scheduled_jobs() - for scheduled_job in scheduled_jobs: - with atomic(): - scheduled_job.mark_running() - try: - run_aggregation(scheduled_job.period) - scheduled_job.mark_finished() - except Exception as e: - scheduled_job.mark_failed() - # Only send job failure email the first time - if scheduled_job.attempts <= 1: - email_job_owners( - scheduled_job, - "indicator_aggregation/fail_subject.txt", - "indicator_aggregation/fail_message.html", - reason=str(e) - ) - logger.error("Failed executing aggregation job %s: %s", scheduled_job.id, e) - else: - # Send out success email if job previously failed - if scheduled_job.attempts > 1: - email_job_owners( - scheduled_job, - "indicator_aggregation/success_subject.txt", - "indicator_aggregation/success_message.html", - ) + while (scheduled_job := get_scheduled_jobs().first()): + scheduled_job.mark_running() + try: + run_aggregation(scheduled_job.period) + scheduled_job.mark_finished() + except Exception as e: + scheduled_job.mark_failed() + # Only send job failure email the first time + if scheduled_job.attempts <= 1: + email_job_owners( + scheduled_job, + "indicator_aggregation/fail_subject.txt", + "indicator_aggregation/fail_message.html", + reason=str(e) + ) + logger.error("Failed executing aggregation job %s: %s", scheduled_job.id, e) + else: + # Send out success email if job previously failed + if scheduled_job.attempts > 1: + email_job_owners( + scheduled_job, + "indicator_aggregation/success_subject.txt", + "indicator_aggregation/success_message.html", + ) def run_aggregation(period: IndicatorPeriod): diff --git a/akvo/settings/90-finish.conf b/akvo/settings/90-finish.conf index 6e89fd7967..8dc57c2ecf 100644 --- a/akvo/settings/90-finish.conf +++ b/akvo/settings/90-finish.conf @@ -109,3 +109,4 @@ CRONJOBS = [ ('*/5 * * * *', 'django.core.management.call_command', ['perform_iati_checks']), ('0 0 * * *', 'django.core.management.call_command', ['cleanup_untitled_and_unpublished_projects']), ] +CRONTAB_LOCK_JOBS = True From 03c68cd63dce26774a2917653e25e1271de3a4c7 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Tue, 1 Nov 2022 19:50:19 +0700 Subject: [PATCH 24/59] [#5145] Create aggregated actual component --- akvo/rsr/spa/app/components/Flex.jsx | 25 ++++++++ akvo/rsr/spa/app/components/Flex.scss | 12 ++++ akvo/rsr/spa/app/components/Icon.jsx | 14 +++++ akvo/rsr/spa/app/components/index.js | 2 + akvo/rsr/spa/app/images/rsr-alert-circle.svg | 3 + akvo/rsr/spa/app/images/rsr-check-circle.svg | 3 + akvo/rsr/spa/app/images/rsr-repeat.svg | 3 + .../app/modules/program/AggregatedActual.jsx | 60 +++++++++++++++++++ .../spa/app/modules/program/ProgramPeriod.jsx | 33 ++++++++-- .../app/modules/program/ProjectSummary.jsx | 22 ++++++- akvo/rsr/spa/app/modules/program/config.js | 14 +++++ .../rsr/spa/app/modules/program/indicator.jsx | 2 +- akvo/rsr/spa/app/modules/program/styles.scss | 29 +++++++-- akvo/rsr/spa/app/utils/common.scss | 2 + akvo/rsr/spa/app/utils/icons.js | 13 ++++ 15 files changed, 223 insertions(+), 14 deletions(-) create mode 100644 akvo/rsr/spa/app/components/Flex.jsx create mode 100644 akvo/rsr/spa/app/components/Flex.scss create mode 100644 akvo/rsr/spa/app/components/Icon.jsx create mode 100644 akvo/rsr/spa/app/images/rsr-alert-circle.svg create mode 100644 akvo/rsr/spa/app/images/rsr-check-circle.svg create mode 100644 akvo/rsr/spa/app/images/rsr-repeat.svg create mode 100644 akvo/rsr/spa/app/modules/program/AggregatedActual.jsx create mode 100644 akvo/rsr/spa/app/utils/icons.js diff --git a/akvo/rsr/spa/app/components/Flex.jsx b/akvo/rsr/spa/app/components/Flex.jsx new file mode 100644 index 0000000000..03f9904607 --- /dev/null +++ b/akvo/rsr/spa/app/components/Flex.jsx @@ -0,0 +1,25 @@ +import React from 'react' +import classNames from 'classnames' +import './Flex.scss' + +const Flex = ({ children, className = '', ...props }) => { + return ( +
+ {children} +
+ ) +} + +const Col = ({ children, className = '', ...props }) => { + return ( +
+ {children} +
+ ) +} + +Flex.Col = Col + +export default Flex diff --git a/akvo/rsr/spa/app/components/Flex.scss b/akvo/rsr/spa/app/components/Flex.scss new file mode 100644 index 0000000000..857647d04f --- /dev/null +++ b/akvo/rsr/spa/app/components/Flex.scss @@ -0,0 +1,12 @@ +.d-flex { + display: flex; + flex-direction: row; + justify-content: flex-end; + align-items: center; + & > div.col { + margin-right: 9px; + } + div.col.icon { + padding-top: 8px; + } +} diff --git a/akvo/rsr/spa/app/components/Icon.jsx b/akvo/rsr/spa/app/components/Icon.jsx new file mode 100644 index 0000000000..e358657b97 --- /dev/null +++ b/akvo/rsr/spa/app/components/Icon.jsx @@ -0,0 +1,14 @@ +import React from 'react' +import { Icon as AntIcon } from 'antd' +import SVGInline from 'react-svg-inline' +import get from 'lodash/get' +import { icons } from '../utils/icons' + +const Icon = ({ type, ...props }) => { + const customIcon = get(icons, type) + return customIcon + ? + : +} + +export default Icon diff --git a/akvo/rsr/spa/app/components/index.js b/akvo/rsr/spa/app/components/index.js index 41b9e4c4a2..d8119656ea 100644 --- a/akvo/rsr/spa/app/components/index.js +++ b/akvo/rsr/spa/app/components/index.js @@ -5,3 +5,5 @@ export { MobileSlider } from './MobileSlider' export { PrevUpdate } from './PrevUpdate' export { DeclinePopup } from './DeclinePopup' export { IndicatorItem } from './IndicatorItem' +export { default as Icon } from './Icon' +export { default as Flex } from './Flex' diff --git a/akvo/rsr/spa/app/images/rsr-alert-circle.svg b/akvo/rsr/spa/app/images/rsr-alert-circle.svg new file mode 100644 index 0000000000..5d3600c2f9 --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-alert-circle.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/images/rsr-check-circle.svg b/akvo/rsr/spa/app/images/rsr-check-circle.svg new file mode 100644 index 0000000000..27146b54e3 --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-check-circle.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/images/rsr-repeat.svg b/akvo/rsr/spa/app/images/rsr-repeat.svg new file mode 100644 index 0000000000..6c843d3cd6 --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-repeat.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx new file mode 100644 index 0000000000..19b99fe996 --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -0,0 +1,60 @@ +import React from 'react' +import { Button } from 'antd' +import SimpleMarkdown from 'simple-markdown' +import classNames from 'classnames' + +import { Flex, Icon } from '../../components' +import { setNumberFormat } from '../../utils/misc' +import { popOver, statusIcons } from './config' + +const ContentPopOver = ({ status, callback }) => { + const content = popOver[status] || {} + let description = content?.description || '' + description = description?.replace(':value:', '2 projects') + description = description?.replace(':total:', '85 projects') + const mdParse = SimpleMarkdown.defaultBlockParse + const mdOutput = SimpleMarkdown.defaultOutput + return ( + <> +

{content?.title}

+
+ {mdOutput(mdParse(description))} +
+ {(content?.action && callback) && ( +
+ +
+ )} + + ) +} + +const AggregatedActual = ({ children, ...props }) => { + return ( + + {children} + + ) +} + +const Col = ({ children, icon = false, className = '', ...props }) => ( + + {children} + +) + +const Value = ({ children, ...props }) => {setNumberFormat(children)} + +const IconComponent = ({ status, width = '24px', height = '24px' }) => { + const iconType = statusIcons[status] || null + return +} + +AggregatedActual.Col = Col +AggregatedActual.Value = Value +AggregatedActual.Popover = ContentPopOver +AggregatedActual.Icon = IconComponent + +export default AggregatedActual diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index 59db9a13d8..dccee778dd 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -13,6 +13,7 @@ import Comments from './Comments' import ExpandIcon from './ExpandIcon' import ProjectSummary from './ProjectSummary' import Disaggregations from './Disaggregations' +import AggregatedActual from './AggregatedActual' const { Panel } = Collapse const { Option } = Select @@ -85,8 +86,17 @@ const PeriodHeader = ({ /> )}
-
aggregated actual value
- {String(actualValue).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} +
aggregated actual
+ + + + + + + {actualValue} + + + {targetsAt && targetsAt === 'period' && targetValue > 0 && ( of {setNumberFormat(countryFilter.length > 0 ? aggFilteredTotalTarget : targetValue)} target @@ -274,10 +284,21 @@ const ProgramPeriod = ({

- {indicatorType === 'quantitative' && [ - {String(subproject.actualValue).replace(/\B(?=(\d{3})+(?!\d))/g, ',')}, - {Math.round((subproject.actualValue / project.actualValue) * 100 * 10) / 10}% - ]} + {indicatorType === 'quantitative' && ( + <> + + + + + + + {subproject.actualValue} + + + + {Math.round((subproject.actualValue / project.actualValue) * 100 * 10) / 10}% + + )} {(indicatorType === 'qualitative' && scoreOptions != null) && (
Score {subproject.scoreIndex + 1}
)} diff --git a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx index de1ab0f1ab..6534cc47d2 100644 --- a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx +++ b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx @@ -1,6 +1,7 @@ import React from 'react' import moment from 'moment' import { Icon, Tooltip } from 'antd' +import AggregatedActual from './AggregatedActual' const getAggregatedUpdatesLength = (updates, contributors) => { let total = 0 @@ -11,6 +12,23 @@ const getAggregatedUpdatesLength = (updates, contributors) => { return total } +const ActualValue = ({ value, status }) => { + return ( + + + + + + + + + {value} + + + + ) +} + const ProjectSummary = ({ _index, indicatorType, @@ -36,7 +54,7 @@ const ProjectSummary = ({ openedItem === _index ? (
- {String(updatesValue).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} + {actualValue > 0 && {Math.round(((updatesValue) / actualValue) * 100 * 10) / 10}%} {updates.length > 0 &&
@@ -51,7 +69,7 @@ const ProjectSummary = ({ : (
- {String(actualValue).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} + {aggFilteredTotal > 0 && {Math.round((actualValue / aggFilteredTotal) * 100 * 10) / 10}%}
) diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index c4fb12784b..4a14d4f0f6 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -21,3 +21,17 @@ export const sizes = { height: 162 + 80 } } + +export const statusIcons = { + repeat: 'rsr.repeat', + finished: 'rsr.circle.checked', + failed: 'rsr.circle.alert', +} + +export const popOver = { + failed: { + title: 'Cron Job Failed', + description: '**:value:** out of **:total:** failed to update', + action: 'view all', + }, +} diff --git a/akvo/rsr/spa/app/modules/program/indicator.jsx b/akvo/rsr/spa/app/modules/program/indicator.jsx index 0b6e48cf0f..13429926e6 100644 --- a/akvo/rsr/spa/app/modules/program/indicator.jsx +++ b/akvo/rsr/spa/app/modules/program/indicator.jsx @@ -74,7 +74,7 @@ const Indicator = ({
-
aggregated actual value
+
aggregated actual
{setNumberFormat(sumActualValue)}
of {indicator.targetValue} target diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index 9bdd6239f0..b4fcd77ad9 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -421,16 +421,19 @@ .stat{ margin-top: -2px; margin-left: 20px; - &.value{ - color: #43998f; - } .label{ font-size: 11px; text-transform: uppercase; font-weight: 300; } - &>b{ - font-size: 18px; + .d-flex { + .icon .failed { + color: $error700; + } + b { + font-size: 18px; + color: #43998f; + } } } .charts{ @@ -1117,3 +1120,19 @@ } } } + +.ant-popover { + max-width: 206px; + &.failed .ant-popover-inner-content{ + background-color: $error50; + } + .description { + margin-bottom: 2.5rem; + } + .action > .ant-btn.ant-btn-link { + width: 100%; + text-align: right; + font-weight: 700; + text-transform: capitalize; + } +} diff --git a/akvo/rsr/spa/app/utils/common.scss b/akvo/rsr/spa/app/utils/common.scss index b2b2656b72..08a16b6ee2 100644 --- a/akvo/rsr/spa/app/utils/common.scss +++ b/akvo/rsr/spa/app/utils/common.scss @@ -5,6 +5,8 @@ $g2colors: #0c7b93, #00a8cc, #ddba01, #b0a160, #434e52, #5b8c85, #8d6e63, #ea908 $g3colors: #2c498b, #35619b, #3e78ab, #4891bb, #52aacb, #6abdd0, #8ecccc, #b4dbcb, #8ca26f; $g4colors: #B40815, #CC0516, #EC1817, #F9532F, #FF9B58, #D62A0B; $submitted: rgb(24, 144, 255); +$error50: #FEF3F2; +$error700: #B42318; @mixin disaggregation-bar-colors{ &:nth-child(1) .disaggregations-bar .dsg-item{ diff --git a/akvo/rsr/spa/app/utils/icons.js b/akvo/rsr/spa/app/utils/icons.js new file mode 100644 index 0000000000..eca0b434f4 --- /dev/null +++ b/akvo/rsr/spa/app/utils/icons.js @@ -0,0 +1,13 @@ +import rsrAlertCircle from '../images/rsr-alert-circle.svg' +import rsrCheckCircle from '../images/rsr-check-circle.svg' +import rsrRepeat from '../images/rsr-repeat.svg' + +export const icons = { + rsr: { + circle: { + alert: rsrAlertCircle, + check: rsrCheckCircle + }, + repeat: rsrRepeat + } +} From f28ce03f50e87e6266ecdfb6053adfac1df88ada Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Nov 2022 15:37:13 +0100 Subject: [PATCH 25/59] feat: Add /rest/v1/jobs/indicator_period_aggregation endpoint Allows users to access IndicatorPeriodAggregationJob's #5145: Feature Request: UI for aggregation tasks --- akvo/rest/serializers/__init__.py | 2 + .../indicator_period_aggregation_job.py | 15 ++++ akvo/rest/urls.py | 1 + akvo/rest/views/__init__.py | 2 + .../views/indicator_period_aggregation_job.py | 21 +++++ akvo/rsr/models/__init__.py | 3 + akvo/rsr/tests/rest/jobs/__init__.py | 0 .../jobs/test_indicator_period_aggregation.py | 88 +++++++++++++++++++ .../tests/usecases/jobs/test_aggregation.py | 30 ++++--- 9 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 akvo/rest/serializers/indicator_period_aggregation_job.py create mode 100644 akvo/rest/views/indicator_period_aggregation_job.py create mode 100644 akvo/rsr/tests/rest/jobs/__init__.py create mode 100644 akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py diff --git a/akvo/rest/serializers/__init__.py b/akvo/rest/serializers/__init__.py index 5ce63cb978..f7ee316415 100644 --- a/akvo/rest/serializers/__init__.py +++ b/akvo/rest/serializers/__init__.py @@ -30,6 +30,7 @@ IndicatorFrameworkNotSoLiteSerializer) from .indicator_label import IndicatorLabelSerializer from .indicator_period import IndicatorPeriodSerializer, IndicatorPeriodFrameworkSerializer +from .indicator_period_aggregation_job import IndicatorPeriodAggregationJobSerializer from .indicator_period_data import (IndicatorPeriodDataSerializer, IndicatorPeriodDataFrameworkSerializer, IndicatorPeriodDataCommentSerializer) @@ -143,6 +144,7 @@ 'IndicatorFrameworkSerializer', 'IndicatorLabelSerializer', 'IndicatorPeriodActualLocationSerializer', + 'IndicatorPeriodAggregationJobSerializer', 'IndicatorPeriodDataCommentSerializer', 'IndicatorPeriodDataFrameworkSerializer', 'IndicatorPeriodDataSerializer', diff --git a/akvo/rest/serializers/indicator_period_aggregation_job.py b/akvo/rest/serializers/indicator_period_aggregation_job.py new file mode 100644 index 0000000000..4f7c5590a1 --- /dev/null +++ b/akvo/rest/serializers/indicator_period_aggregation_job.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Akvo RSR is covered by the GNU Affero General Public License. +# See more details in the license.txt file located at the root folder of the Akvo RSR module. +# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. + + +from akvo.rest.serializers.rsr_serializer import BaseRSRSerializer +from akvo.rsr.models import IndicatorPeriodAggregationJob + + +class IndicatorPeriodAggregationJobSerializer(BaseRSRSerializer): + class Meta: + model = IndicatorPeriodAggregationJob + fields = "__all__" diff --git a/akvo/rest/urls.py b/akvo/rest/urls.py index a24407fbc9..ed62bd3583 100644 --- a/akvo/rest/urls.py +++ b/akvo/rest/urls.py @@ -37,6 +37,7 @@ router.register(r'(?P(v1))/iati_check', views.IatiCheckViewSet) router.register(r'(?P(v1))/iati_export', views.IatiExportViewSet) router.register(r'(?P(v1))/indicator', views.IndicatorViewSet) +router.register(r'(?P(v1))/jobs/indicator_period_aggregation', views.IndicatorPeriodAggregationJobViewSet) router.register(r'(?P(v1))/dimension_name', views.IndicatorDimensionNameViewSet) router.register(r'(?P(v1))/dimension_value', views.IndicatorDimensionValueViewSet) router.register(r'(?P(v1))/indicator_custom_field', views.IndicatorCustomFieldViewSet) diff --git a/akvo/rest/views/__init__.py b/akvo/rest/views/__init__.py index ade8858f50..748002f7f5 100644 --- a/akvo/rest/views/__init__.py +++ b/akvo/rest/views/__init__.py @@ -31,6 +31,7 @@ from .indicator_dimension_name import IndicatorDimensionNameViewSet from .indicator_dimension_value import IndicatorDimensionValueViewSet from .indicator_label import IndicatorLabelViewSet +from .indicator_period_aggregation_job import IndicatorPeriodAggregationJobViewSet from .indicator_period_label import IndicatorPeriodLabelViewSet, project_period_labels from .indicator_period import (IndicatorPeriodViewSet, IndicatorPeriodFrameworkViewSet, set_periods_locked, bulk_add_periods, bulk_remove_periods) @@ -138,6 +139,7 @@ 'IatiCheckViewSet', 'IatiExportViewSet', 'IndicatorViewSet', + 'IndicatorPeriodAggregationJobViewSet', 'IndicatorCustomFieldViewSet', 'IndicatorCustomValueViewSet', 'IndicatorDimensionNameViewSet', diff --git a/akvo/rest/views/indicator_period_aggregation_job.py b/akvo/rest/views/indicator_period_aggregation_job.py new file mode 100644 index 0000000000..2904db8f8a --- /dev/null +++ b/akvo/rest/views/indicator_period_aggregation_job.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# Akvo RSR is covered by the GNU Affero General Public License. +# See more details in the license.txt file located at the root folder of the Akvo RSR module. +# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. +from rest_framework.permissions import IsAuthenticated + +from akvo.rsr.models import IndicatorPeriodAggregationJob + +from ..serializers import IndicatorPeriodAggregationJobSerializer +from ..viewsets import ReadOnlyPublicProjectViewSet, SafeMethodsPermissions + + +class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet): + queryset = IndicatorPeriodAggregationJob.objects.all() + serializer_class = IndicatorPeriodAggregationJobSerializer + project_relation = "program__" + ordering = ["updated_at"] + + # These are login only resources that shouldn't be interesting to the public + permission_classes = (SafeMethodsPermissions, IsAuthenticated) diff --git a/akvo/rsr/models/__init__.py b/akvo/rsr/models/__init__.py index 86c43fc905..8221abfd48 100644 --- a/akvo/rsr/models/__init__.py +++ b/akvo/rsr/models/__init__.py @@ -265,6 +265,9 @@ rules.add_perm('rsr.change_indicatorperioddisaggregation', is_rsr_admin | is_org_admin | is_org_me_manager_or_project_editor) rules.add_perm('rsr.view_indicatorperioddisaggregation', is_org_enumerator) +rules.add_perm('rsr.change_indicatorperiodaggregationjob', is_rsr_admin | is_org_admin | is_org_me_manager_or_project_editor) +rules.add_perm('rsr.view_indicatorperiodaggregationjob', is_org_enumerator) + rules.add_perm('rsr.view_indicatorperioddata', is_rsr_admin | is_org_admin | is_org_me_manager) rules.add_perm( 'rsr.add_indicatorperioddata', diff --git a/akvo/rsr/tests/rest/jobs/__init__.py b/akvo/rsr/tests/rest/jobs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py new file mode 100644 index 0000000000..94d4caa76e --- /dev/null +++ b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +""" +Akvo RSR is covered by the GNU Affero General Public License. + +See more details in the license.txt file located at the root folder of the Akvo RSR module. +For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. +""" + +from rest_framework.status import HTTP_200_OK, HTTP_403_FORBIDDEN + +from akvo.rsr.models import IndicatorPeriodAggregationJob +from akvo.rsr.permissions import GROUP_NAME_ME_MANAGERS +from akvo.rsr.tests.base import BaseTestCase +from akvo.rsr.tests.usecases.jobs.test_aggregation import AggregationJobBaseTests + + +class AnonymousUserTestCase(BaseTestCase): + + def test_anonymous_user(self): + """Shouldn't be able to access any resources even if the project is public""" + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json") + self.assertEqual(response.status_code, HTTP_403_FORBIDDEN) + + +class AuthorizationTestCase(AggregationJobBaseTests): + """Tests accessing indicator period aggregation job REST endpoints""" + + def setUp(self): + super().setUp() + + # Create private project in the default org + self.private_user = self.create_user("private@akvo.org", "password", is_superuser=False) + self.private_project = self.create_project(f"Super private project", public=False) + self.private_project.set_reporting_org(self.org) + self.make_employment(self.private_user, self.org, GROUP_NAME_ME_MANAGERS) + + self.private_result, self.private_indicator, self.private_period = \ + self._make_results_framework(self.private_project) + self.private_job = IndicatorPeriodAggregationJob.objects.create( + period=self.private_period, program=self.private_project + ) + + # Create private project in another org + self.other_private_user = self.create_user("other_private@akvo.org", "password", is_superuser=False) + self.other_private_project, self.other_private_org = self._make_project("Private", public=False) + self.make_employment(self.other_private_user, self.other_private_org, GROUP_NAME_ME_MANAGERS) + + self.other_private_result, self.other_private_indicator, self.other_private_period = \ + self._make_results_framework(self.other_private_project) + self.other_private_job = IndicatorPeriodAggregationJob.objects.create( + period=self.other_private_period, program=self.other_private_project + ) + + def test_super_user(self): + """Super users be able to access all jobs""" + self.c.login(username=self.user.username, password="password") + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json") + self.assertEqual(response.status_code, HTTP_200_OK) + + data = response.json() + self.assertEqual(data.get("count"), 3) + + results = data.get("results") + self.assertEqual( + {result["id"] for result in results}, + {self.job.id, self.private_job.id, self.other_private_job.id}, + ) + + def test_private_user(self): + """Test a private user accessing jobs from the default org""" + self.c.login(username=self.private_user.username, password="password") + self._test_private_user_access({self.job.id, self.private_job.id}) + + def test_other_private_user(self): + """Test a private user accessing jobs from the private org""" + self.c.login(username=self.other_private_user.username, password="password") + self._test_private_user_access({self.job.id, self.other_private_job.id}) + + def _test_private_user_access(self, expected_job_id_set): + """ + Private users should only be able to access jobs of their private projects and that of public ones + """ + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json") + self.assertEqual(response.status_code, HTTP_200_OK) + data = response.json() + self.assertEqual(data.get("count"), 2) + self.assertEqual({result["id"] for result in data["results"]}, expected_job_id_set) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index 352757bba1..d3309414e4 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -22,24 +22,34 @@ def setUp(self): password="password" ) - self.project = self.create_project('Test project') - self.org = self.create_organisation("Test org") - self.project.set_reporting_org(self.org) + self.project, self.org = self._make_project("Main") # Create results framework - self.result = Result.objects.create( - project=self.project, title='Result #1', type='1' + self.result, self.indicator, self.period = self._make_results_framework(self.project) + + self.job = IndicatorPeriodAggregationJob.objects.create(period=self.period, program=self.project) + + def _make_project(self, name, public=True): + project = self.create_project(f"{name} project", public=public) + org = self.create_organisation(f"{name} org") + project.set_reporting_org(org) + + return project, org + + def _make_results_framework(self, project): + result = Result.objects.create( + project=project, title='Result #1', type='1' ) - self.indicator = Indicator.objects.create( - result=self.result, title='Indicator #1' + indicator = Indicator.objects.create( + result=result, title='Indicator #1' ) - self.period = IndicatorPeriod.objects.create( - indicator=self.indicator, + period = IndicatorPeriod.objects.create( + indicator=indicator, period_start=datetime.date.today(), period_end=datetime.date.today() + datetime.timedelta(days=1), target_value="100" ) - self.job = IndicatorPeriodAggregationJob.objects.create(period=self.period, program=self.project) + return result, indicator, period class JobStatusTestCase(AggregationJobBaseTests): From 684e09831c4c0994bf898ff62a1f707144ee6a22 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 1 Nov 2022 16:20:04 +0100 Subject: [PATCH 26/59] test: /rest/v1/jobs/indicator_period_aggregation filters Passing &filter={...} query param #5145: Feature Request: UI for aggregation tasks --- .../views/indicator_period_aggregation_job.py | 3 ++ .../jobs/test_indicator_period_aggregation.py | 42 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/akvo/rest/views/indicator_period_aggregation_job.py b/akvo/rest/views/indicator_period_aggregation_job.py index 2904db8f8a..6a309c5326 100644 --- a/akvo/rest/views/indicator_period_aggregation_job.py +++ b/akvo/rest/views/indicator_period_aggregation_job.py @@ -3,9 +3,11 @@ # Akvo RSR is covered by the GNU Affero General Public License. # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. +from rest_framework import filters from rest_framework.permissions import IsAuthenticated from akvo.rsr.models import IndicatorPeriodAggregationJob +from ..filters import RSRGenericFilterBackend from ..serializers import IndicatorPeriodAggregationJobSerializer from ..viewsets import ReadOnlyPublicProjectViewSet, SafeMethodsPermissions @@ -16,6 +18,7 @@ class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet): serializer_class = IndicatorPeriodAggregationJobSerializer project_relation = "program__" ordering = ["updated_at"] + filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,) # These are login only resources that shouldn't be interesting to the public permission_classes = (SafeMethodsPermissions, IsAuthenticated) diff --git a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py index 94d4caa76e..55f990c129 100644 --- a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py +++ b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py @@ -6,6 +6,7 @@ See more details in the license.txt file located at the root folder of the Akvo RSR module. For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. """ +from unittest.mock import ANY from rest_framework.status import HTTP_200_OK, HTTP_403_FORBIDDEN @@ -23,7 +24,7 @@ def test_anonymous_user(self): self.assertEqual(response.status_code, HTTP_403_FORBIDDEN) -class AuthorizationTestCase(AggregationJobBaseTests): +class EndpointTestCase(AggregationJobBaseTests): """Tests accessing indicator period aggregation job REST endpoints""" def setUp(self): @@ -31,7 +32,7 @@ def setUp(self): # Create private project in the default org self.private_user = self.create_user("private@akvo.org", "password", is_superuser=False) - self.private_project = self.create_project(f"Super private project", public=False) + self.private_project = self.create_project("Super private project", public=False) self.private_project.set_reporting_org(self.org) self.make_employment(self.private_user, self.org, GROUP_NAME_ME_MANAGERS) @@ -86,3 +87,40 @@ def _test_private_user_access(self, expected_job_id_set): data = response.json() self.assertEqual(data.get("count"), 2) self.assertEqual({result["id"] for result in data["results"]}, expected_job_id_set) + + def test_filter_by_program(self): + self.c.login(username=self.user.username, password="password") + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json&filter={'program_id':%s}" % ( + self.project.id + )) + + self.assertEqual(response.status_code, HTTP_200_OK) + + data = response.json() + self.assertEqual(data["count"], 1) + + self.assertEqual(data["results"][0]["id"], self.job.id) + + def test_filter_by_status(self): + self.job.status = IndicatorPeriodAggregationJob.Status.FINISHED + self.job.save() + + self.c.login(username=self.user.username, password="password") + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json&filter={'status':'%s'}" % ( + self.job.status.FINISHED + )) + self.assertEqual((response.status_code, response.json()), (HTTP_200_OK, ANY)) + + data = response.json() + self.assertEqual(data["count"], 1) + + self.assertEqual(data["results"][0]["id"], self.job.id) + + def test_get_by_id(self): + self.c.login(username=self.user.username, password="password") + response = self.c.get(f"/rest/v1/jobs/indicator_period_aggregation/{self.private_job.id}/?format=json") + + self.assertEqual(response.status_code, HTTP_200_OK) + + data = response.json() + self.assertEqual(data["id"], self.private_job.id) From fa260aaf1bf5d0ef32f20a8a435421f42345b095 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Wed, 2 Nov 2022 19:39:33 +0700 Subject: [PATCH 27/59] [#5145] Get jobs status by period id --- akvo/rsr/spa/app/components/Flex.scss | 1 + akvo/rsr/spa/app/components/index.js | 2 - akvo/rsr/spa/app/images/rsr-clock.svg | 3 + akvo/rsr/spa/app/images/rsr-loader.svg | 3 + .../spa/app/modules/program/ActualValue.jsx | 42 ++++++++++ .../app/modules/program/AggregatedActual.jsx | 78 ++++++----------- .../spa/app/modules/program/Aggregation.jsx | 84 +++++++++++++++++++ .../spa/app/modules/program/ProgramPeriod.jsx | 33 +++----- .../app/modules/program/ProjectSummary.jsx | 29 ++----- akvo/rsr/spa/app/modules/program/config.js | 26 ++++-- akvo/rsr/spa/app/modules/program/services.js | 20 +++++ akvo/rsr/spa/app/modules/program/styles.scss | 21 ++++- akvo/rsr/spa/app/utils/common.scss | 1 + akvo/rsr/spa/app/utils/icons.js | 6 +- 14 files changed, 240 insertions(+), 109 deletions(-) create mode 100644 akvo/rsr/spa/app/images/rsr-clock.svg create mode 100644 akvo/rsr/spa/app/images/rsr-loader.svg create mode 100644 akvo/rsr/spa/app/modules/program/ActualValue.jsx create mode 100644 akvo/rsr/spa/app/modules/program/Aggregation.jsx create mode 100644 akvo/rsr/spa/app/modules/program/services.js diff --git a/akvo/rsr/spa/app/components/Flex.scss b/akvo/rsr/spa/app/components/Flex.scss index 857647d04f..3a76da3457 100644 --- a/akvo/rsr/spa/app/components/Flex.scss +++ b/akvo/rsr/spa/app/components/Flex.scss @@ -8,5 +8,6 @@ } div.col.icon { padding-top: 8px; + cursor: pointer; } } diff --git a/akvo/rsr/spa/app/components/index.js b/akvo/rsr/spa/app/components/index.js index d8119656ea..41b9e4c4a2 100644 --- a/akvo/rsr/spa/app/components/index.js +++ b/akvo/rsr/spa/app/components/index.js @@ -5,5 +5,3 @@ export { MobileSlider } from './MobileSlider' export { PrevUpdate } from './PrevUpdate' export { DeclinePopup } from './DeclinePopup' export { IndicatorItem } from './IndicatorItem' -export { default as Icon } from './Icon' -export { default as Flex } from './Flex' diff --git a/akvo/rsr/spa/app/images/rsr-clock.svg b/akvo/rsr/spa/app/images/rsr-clock.svg new file mode 100644 index 0000000000..69699c0c8f --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-clock.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/images/rsr-loader.svg b/akvo/rsr/spa/app/images/rsr-loader.svg new file mode 100644 index 0000000000..22cc17661c --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-loader.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/modules/program/ActualValue.jsx b/akvo/rsr/spa/app/modules/program/ActualValue.jsx new file mode 100644 index 0000000000..71736f466d --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/ActualValue.jsx @@ -0,0 +1,42 @@ +import { Spin } from 'antd' +import React, { useState } from 'react' +import Icon from '../../components/Icon' +import Aggregation from './Aggregation' +import { popOver, statusIcons } from './config' +import { getJobStatusByPeriod } from './services' + +const ActualValue = ({ + actualValue, + periodId, +}) => { + const [job, setJob] = useState(null) + getJobStatusByPeriod(periodId) + ?.then((res) => { + setJob(res?.shift()) + }) + ?.catch(() => setJob({ status: 'FAILED' })) + const title = popOver[job?.status] ? popOver[job.status]?.title : null + const iconType = statusIcons[job?.status] || null + return ( + + + { + (job === null) + ? } /> + : ( + + + + ) + } + + + + {actualValue || '...'} + + + + ) +} + +export default ActualValue diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx index 19b99fe996..1fbf637cac 100644 --- a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -1,60 +1,30 @@ import React from 'react' -import { Button } from 'antd' -import SimpleMarkdown from 'simple-markdown' -import classNames from 'classnames' - -import { Flex, Icon } from '../../components' -import { setNumberFormat } from '../../utils/misc' -import { popOver, statusIcons } from './config' - -const ContentPopOver = ({ status, callback }) => { - const content = popOver[status] || {} - let description = content?.description || '' - description = description?.replace(':value:', '2 projects') - description = description?.replace(':total:', '85 projects') - const mdParse = SimpleMarkdown.defaultBlockParse - const mdOutput = SimpleMarkdown.defaultOutput - return ( - <> -

{content?.title}

-
- {mdOutput(mdParse(description))} -
- {(content?.action && callback) && ( -
- -
- )} - - ) -} - -const AggregatedActual = ({ children, ...props }) => { +import Icon from '../../components/Icon' +import Aggregation from './Aggregation' +import { statusIcons } from './config' + +const AggregatedActual = ({ + value, + status, + amount, + total, + callback, +}) => { + const iconType = statusIcons[status] || statusIcons.FINISHED return ( - - {children} - + + + + + + + + + {value} + + + ) } -const Col = ({ children, icon = false, className = '', ...props }) => ( - - {children} - -) - -const Value = ({ children, ...props }) => {setNumberFormat(children)} - -const IconComponent = ({ status, width = '24px', height = '24px' }) => { - const iconType = statusIcons[status] || null - return -} - -AggregatedActual.Col = Col -AggregatedActual.Value = Value -AggregatedActual.Popover = ContentPopOver -AggregatedActual.Icon = IconComponent - export default AggregatedActual diff --git a/akvo/rsr/spa/app/modules/program/Aggregation.jsx b/akvo/rsr/spa/app/modules/program/Aggregation.jsx new file mode 100644 index 0000000000..48d1630275 --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/Aggregation.jsx @@ -0,0 +1,84 @@ +import React from 'react' +import { Button, Popover, Tooltip } from 'antd' +import SimpleMarkdown from 'simple-markdown' +import classNames from 'classnames' +import { useTranslation } from 'react-i18next' + +import Flex from '../../components/Flex' +import { setNumberFormat } from '../../utils/misc' +import { popOver } from './config' + +const ContentPopOver = ({ status, callback, amount = 2, total = 85 }) => { + const { t: trans } = useTranslation() + const content = popOver[status] || {} + let description = content?.description || '' + description = description?.replace(':value:', `${amount} ${trans('contributor_s', { count: amount })}`) + description = description?.replace(':total:', `${total} ${trans('contributor_s', { count: total })}`) + const mdParse = SimpleMarkdown.defaultBlockParse + const mdOutput = SimpleMarkdown.defaultOutput + return ( + <> +

{content?.title}

+
+ {mdOutput(mdParse(description))} +
+ {(content?.action && callback) && ( +
+ +
+ )} + + ) +} + +const Aggregation = ({ children, ...props }) => { + return ( + + {children} + + ) +} + +const Col = ({ children, icon = false, className = '', ...props }) => ( + + {children} + +) + +const Value = ({ children, ...props }) => {setNumberFormat(children)} + +const TooltipComponent = ({ children, placement = 'top', title = 'Hello world', ...props }) => { + return ( + + {children} + + ) +} + +const PopoverComponent = ({ children, status, callback, amount, total, placement = 'top', ...props }) => { + return ( + + )} + overlayClassName={status} {...props} + > + {children} + + ) +} + +Aggregation.Col = Col +Aggregation.Value = Value +Aggregation.Tooltip = TooltipComponent +Aggregation.Popover = PopoverComponent + +export default Aggregation diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index dccee778dd..11e40d3e29 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -1,6 +1,6 @@ /* global document */ import React from 'react' -import { Collapse, Icon, Select } from 'antd' +import { Collapse, Select } from 'antd' import moment from 'moment' import classNames from 'classnames' import { useTranslation } from 'react-i18next' @@ -13,6 +13,8 @@ import Comments from './Comments' import ExpandIcon from './ExpandIcon' import ProjectSummary from './ProjectSummary' import Disaggregations from './Disaggregations' +import Icon from '../../components/Icon' +import ActualValue from './ActualValue' import AggregatedActual from './AggregatedActual' const { Panel } = Collapse @@ -64,6 +66,7 @@ const PeriodHeader = ({ disaggregationTargets }) => { const { t } = useTranslation() + const handleOnClickIcon = () => console.log('open collapse') return ( <>
@@ -87,16 +90,13 @@ const PeriodHeader = ({ )}
aggregated actual
- - - - - - - {actualValue} - - - + {targetsAt && targetsAt === 'period' && targetValue > 0 && ( of {setNumberFormat(countryFilter.length > 0 ? aggFilteredTotalTarget : targetValue)} target @@ -286,16 +286,7 @@ const ProgramPeriod = ({
{indicatorType === 'quantitative' && ( <> - - - - - - - {subproject.actualValue} - - - + {Math.round((subproject.actualValue / project.actualValue) * 100 * 10) / 10}% )} diff --git a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx index 6534cc47d2..160a85f848 100644 --- a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx +++ b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx @@ -1,7 +1,8 @@ import React from 'react' import moment from 'moment' -import { Icon, Tooltip } from 'antd' -import AggregatedActual from './AggregatedActual' +import { Tooltip } from 'antd' +import Icon from '../../components/Icon' +import ActualValue from './ActualValue' const getAggregatedUpdatesLength = (updates, contributors) => { let total = 0 @@ -12,23 +13,6 @@ const getAggregatedUpdatesLength = (updates, contributors) => { return total } -const ActualValue = ({ value, status }) => { - return ( - - - - - - - - - {value} - - - - ) -} - const ProjectSummary = ({ _index, indicatorType, @@ -39,7 +23,8 @@ const ProjectSummary = ({ actualValue, updatesValue, updates, - contributors + contributors, + periodId, }) => { if (indicatorType === 'quantitative') { return ( @@ -54,7 +39,7 @@ const ProjectSummary = ({ openedItem === _index ? (
- + {actualValue > 0 && {Math.round(((updatesValue) / actualValue) * 100 * 10) / 10}%} {updates.length > 0 &&
@@ -69,7 +54,7 @@ const ProjectSummary = ({ : (
- + {aggFilteredTotal > 0 && {Math.round((actualValue / aggFilteredTotal) * 100 * 10) / 10}%}
) diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index 4a14d4f0f6..e0460e4ae1 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -23,15 +23,29 @@ export const sizes = { } export const statusIcons = { - repeat: 'rsr.repeat', - finished: 'rsr.circle.checked', - failed: 'rsr.circle.alert', + RETRY: 'rsr.repeat', + SCHEDULED: 'rsr.clock', + RUNNING: 'rsr.loader', + FINISHED: 'rsr.circle.check', + FAILED: 'rsr.circle.alert', } export const popOver = { - failed: { + SCHEDULED: { + title: 'Cron Job Scheduled', + description: 'Aggregation is scheduled', + }, + RUNNING: { + title: 'Cron Job Running', + description: 'Aggregation in progress', + }, + FINISHED: { + title: 'Cron Job Finished', + description: 'Aggregated actual is valid', + }, + FAILED: { title: 'Cron Job Failed', description: '**:value:** out of **:total:** failed to update', - action: 'view all', - }, + action: 'view all' + } } diff --git a/akvo/rsr/spa/app/modules/program/services.js b/akvo/rsr/spa/app/modules/program/services.js new file mode 100644 index 0000000000..645eb8a4fb --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/services.js @@ -0,0 +1,20 @@ +import api from '../../utils/api' + +export const getAggregationJobsApi = async (id, page = 1) => { + /** + * Choices are: attempts, id, period, period_id, pid, program, program_id, status, updated_at + * ?format=json&filter={'program_id':9062} + */ + const response = await api.get(`/jobs/indicator_period_aggregation/?page=${page}&program_id=${id}&format=json`) + const { results, next } = response.data + if (next) { + return results?.concat(await getAggregationJobsApi(id, page + 1)) + } + return results +} + +export const getJobStatusByPeriod = async (period) => { + const response = await api.get(`/jobs/indicator_period_aggregation/?format=json&filter={'period': ${period}}`) + const { results } = response.data + return results +} diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index b4fcd77ad9..3139ccd8ab 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -427,7 +427,7 @@ font-weight: 300; } .d-flex { - .icon .failed { + .icon .FAILED { color: $error700; } b { @@ -622,6 +622,14 @@ display: block; } } + .d-flex .col.icon { + &> .FAILED { + color: $error700; + } + &> .FINISHED { + color: $primary700; + } + } } position: relative; .total{ @@ -794,7 +802,6 @@ justify-content: center; display: flex; align-items: center; - color: #fff; flex-direction: column; &.score{ position: absolute; @@ -823,6 +830,14 @@ display: block; } } + .d-flex .col.icon { + &> .FAILED { + color: $error700; + } + &> .FINISHED { + color: $primary700; + } + } } .updates{ width: 100%; @@ -1123,7 +1138,7 @@ .ant-popover { max-width: 206px; - &.failed .ant-popover-inner-content{ + &.FAILED .ant-popover-inner-content{ background-color: $error50; } .description { diff --git a/akvo/rsr/spa/app/utils/common.scss b/akvo/rsr/spa/app/utils/common.scss index 08a16b6ee2..8fa65dd456 100644 --- a/akvo/rsr/spa/app/utils/common.scss +++ b/akvo/rsr/spa/app/utils/common.scss @@ -7,6 +7,7 @@ $g4colors: #B40815, #CC0516, #EC1817, #F9532F, #FF9B58, #D62A0B; $submitted: rgb(24, 144, 255); $error50: #FEF3F2; $error700: #B42318; +$primary700: #218EFC; @mixin disaggregation-bar-colors{ &:nth-child(1) .disaggregations-bar .dsg-item{ diff --git a/akvo/rsr/spa/app/utils/icons.js b/akvo/rsr/spa/app/utils/icons.js index eca0b434f4..3fd582f0a0 100644 --- a/akvo/rsr/spa/app/utils/icons.js +++ b/akvo/rsr/spa/app/utils/icons.js @@ -1,6 +1,8 @@ import rsrAlertCircle from '../images/rsr-alert-circle.svg' import rsrCheckCircle from '../images/rsr-check-circle.svg' import rsrRepeat from '../images/rsr-repeat.svg' +import rsrLoader from '../images/rsr-loader.svg' +import rsrClock from '../images/rsr-clock.svg' export const icons = { rsr: { @@ -8,6 +10,8 @@ export const icons = { alert: rsrAlertCircle, check: rsrCheckCircle }, - repeat: rsrRepeat + repeat: rsrRepeat, + loader: rsrLoader, + clock: rsrClock, } } From 32c4f843539dbdaf56509fea6a4d144ee15f7650 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Nov 2022 14:02:07 +0100 Subject: [PATCH 28/59] test: Add permission test for job viewset #5145: Feature Request: UI for aggregation tasks --- akvo/rsr/tests/rest/test_permissions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/akvo/rsr/tests/rest/test_permissions.py b/akvo/rsr/tests/rest/test_permissions.py index e9884afd4b..3f01d51ad5 100644 --- a/akvo/rsr/tests/rest/test_permissions.py +++ b/akvo/rsr/tests/rest/test_permissions.py @@ -529,6 +529,12 @@ def populate_model_map(): 'project_relation': 'indicator__result__project__' } + # one indicator period per indicator + model_map[M.IndicatorPeriodAggregationJob] = { + 'group_count': group_count(8, 2, 6, 4), + 'project_relation': 'period__indicator__result__project__' + } + # one indicator period actual location per period # FIXME: change_* permissions weirdness model_map[M.IndicatorPeriodActualLocation] = { From 85d895e83dba5affbc2e575446742ea6884c6182 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Nov 2022 14:09:03 +0100 Subject: [PATCH 29/59] refactor: Use root_period instead of program The UI will have to easily find the jobs of child periods. To make that easier, it's better to retrieve the root period from which one can get the root project. It can also happen that there are master and sub programs. The sub programs can also define their own periods that override those inherited by the parent program. Searching for the parent period allows us to treat this case as well #5145: Feature Request: UI for aggregation tasks --- .../views/indicator_period_aggregation_job.py | 6 ++- akvo/rsr/admin.py | 19 ++++++--- ...dicatorperiodaggregationjob_root_period.py | 41 +++++++++++++++++++ akvo/rsr/models/aggregation_job.py | 16 +++++++- akvo/rsr/models/result/indicator_period.py | 10 +++++ .../jobs/test_indicator_period_aggregation.py | 28 ++++++------- .../tests/usecases/jobs/test_aggregation.py | 2 +- akvo/rsr/usecases/jobs/aggregation.py | 8 ++-- .../indicator_aggregation/fail_message.html | 4 +- .../success_message.html | 4 +- 10 files changed, 106 insertions(+), 32 deletions(-) create mode 100644 akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py diff --git a/akvo/rest/views/indicator_period_aggregation_job.py b/akvo/rest/views/indicator_period_aggregation_job.py index 6a309c5326..d4dd690261 100644 --- a/akvo/rest/views/indicator_period_aggregation_job.py +++ b/akvo/rest/views/indicator_period_aggregation_job.py @@ -14,9 +14,11 @@ class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet): - queryset = IndicatorPeriodAggregationJob.objects.all() + queryset = IndicatorPeriodAggregationJob.objects.all().select_related( + IndicatorPeriodAggregationJob.project_relation[:-2] + ) serializer_class = IndicatorPeriodAggregationJobSerializer - project_relation = "program__" + project_relation = IndicatorPeriodAggregationJob.project_relation ordering = ["updated_at"] filter_backends = (filters.OrderingFilter, RSRGenericFilterBackend,) diff --git a/akvo/rsr/admin.py b/akvo/rsr/admin.py index e1ee6b7f02..faef89ddd9 100644 --- a/akvo/rsr/admin.py +++ b/akvo/rsr/admin.py @@ -946,14 +946,21 @@ def get_queryset(self, request): class IndicatorPeriodAggregationJobAdmin(admin.ModelAdmin): model = apps.get_model('rsr', 'IndicatorPeriodAggregationJob') - list_display = ('status', 'program', 'period', 'indicator_title') + list_display = ('indicator_title', 'status', 'project_title', 'root_project_title', 'period', 'updated_at') list_filter = ('status', ) - search_fields = ('program__title', 'period__indicator__title') - readonly_fields = ('updated_at', 'period', 'program', 'indicator_title') + search_fields = ('period__indicator__result__period__title', 'period__indicator__title') + readonly_fields = ('updated_at', 'period', 'project_title', 'root_project_title', 'indicator_title') - @admin.display(description='Program Title') - def program_title(self, obj): - return obj.program.title + @admin.display(description='Project Title') + def project_title(self, obj): + return self.get_project(obj.period).title + + @admin.display(description='Root project Title') + def root_project_title(self, obj): + return self.get_project(obj.root_period).title + + def get_project(self, period): + return period.indicator.result.project @admin.display(description='Indicator Title') def indicator_title(self, obj): diff --git a/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py b/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py new file mode 100644 index 0000000000..99437c767d --- /dev/null +++ b/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.10 on 2022-11-02 10:27 + +from django.db import migrations, models +import django.db.models.deletion + + +def set_root_period(apps, schema_editor): + IndicatorPeriodAggregationJob = apps.get_model("rsr", "IndicatorPeriodAggregationJob") + jobs = [] + for job in IndicatorPeriodAggregationJob.objects.all(): + job.root_period = job.period.get_root_period() + jobs.append(job) + IndicatorPeriodAggregationJob.objects.bulk_update(jobs, ["root_period"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('rsr', '0221_indicatorperiodaggregationjob'), + ] + operations = [ + migrations.RemoveField( + model_name='indicatorperiodaggregationjob', + name='program', + ), + migrations.AddField( + model_name='indicatorperiodaggregationjob', + name='root_period', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='child_aggregation_jobs', to='rsr.indicatorperiod'), + preserve_default=False, + ), + migrations.RunPython(set_root_period), + + migrations.AlterField( + model_name='indicatorperiodaggregationjob', + name='root_period', + field=models.ForeignKey(null=False, on_delete=django.db.models.deletion.CASCADE, + related_name='child_aggregation_jobs', to='rsr.indicatorperiod'), + preserve_default=False, + ), + ] diff --git a/akvo/rsr/models/aggregation_job.py b/akvo/rsr/models/aggregation_job.py index 19419eed31..c7bd891382 100644 --- a/akvo/rsr/models/aggregation_job.py +++ b/akvo/rsr/models/aggregation_job.py @@ -4,9 +4,23 @@ class IndicatorPeriodAggregationJob(CronJobMixin): + project_relation = "period__indicator__result__project__" + period = models.ForeignKey( "IndicatorPeriod", on_delete=models.CASCADE, related_name="aggregation_jobs" ) - program = models.ForeignKey("Project", on_delete=models.CASCADE) + root_period = models.ForeignKey( + "IndicatorPeriod", + on_delete=models.CASCADE, + related_name="child_aggregation_jobs" + ) + + @property + def project(self): + return self.period.project + + @property + def root_project(self): + return self.root_period.project diff --git a/akvo/rsr/models/result/indicator_period.py b/akvo/rsr/models/result/indicator_period.py index 0cd6e954d6..86fc533299 100644 --- a/akvo/rsr/models/result/indicator_period.py +++ b/akvo/rsr/models/result/indicator_period.py @@ -255,6 +255,16 @@ def adjacent_period(self, next_period=True): return self.indicator.periods.exclude(period_start=None).filter( period_start__lt=self.period_start).order_by('-period_start').first() + def get_root_period(self): + root = self + while root.parent_period: + root = root.parent_period + return root + + @property + def project(self): + return self.indicator.result.project + @property def percent_accomplishment(self): """ diff --git a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py index 55f990c129..f8f560d39d 100644 --- a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py +++ b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py @@ -14,6 +14,7 @@ from akvo.rsr.permissions import GROUP_NAME_ME_MANAGERS from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.tests.usecases.jobs.test_aggregation import AggregationJobBaseTests +from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job class AnonymousUserTestCase(BaseTestCase): @@ -30,17 +31,18 @@ class EndpointTestCase(AggregationJobBaseTests): def setUp(self): super().setUp() - # Create private project in the default org + # Create private child project in the default org self.private_user = self.create_user("private@akvo.org", "password", is_superuser=False) self.private_project = self.create_project("Super private project", public=False) + self.make_parent(self.project, self.private_project) self.private_project.set_reporting_org(self.org) + self.private_project.import_results() self.make_employment(self.private_user, self.org, GROUP_NAME_ME_MANAGERS) - self.private_result, self.private_indicator, self.private_period = \ - self._make_results_framework(self.private_project) - self.private_job = IndicatorPeriodAggregationJob.objects.create( - period=self.private_period, program=self.private_project - ) + self.private_result = self.result.child_results.first() + self.private_indicator = self.indicator.child_indicators.first() + self.private_period = self.period.child_periods.first() + self.private_job = schedule_aggregation_job(self.private_period) # Create private project in another org self.other_private_user = self.create_user("other_private@akvo.org", "password", is_superuser=False) @@ -49,9 +51,7 @@ def setUp(self): self.other_private_result, self.other_private_indicator, self.other_private_period = \ self._make_results_framework(self.other_private_project) - self.other_private_job = IndicatorPeriodAggregationJob.objects.create( - period=self.other_private_period, program=self.other_private_project - ) + self.other_private_job = schedule_aggregation_job(self.other_private_period) def test_super_user(self): """Super users be able to access all jobs""" @@ -85,21 +85,21 @@ def _test_private_user_access(self, expected_job_id_set): response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json") self.assertEqual(response.status_code, HTTP_200_OK) data = response.json() - self.assertEqual(data.get("count"), 2) + self.assertEqual(data.get("count"), len(expected_job_id_set)) self.assertEqual({result["id"] for result in data["results"]}, expected_job_id_set) def test_filter_by_program(self): self.c.login(username=self.user.username, password="password") - response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json&filter={'program_id':%s}" % ( - self.project.id + response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json&filter={'root_period_id':%s}" % ( + self.period.id )) self.assertEqual(response.status_code, HTTP_200_OK) data = response.json() - self.assertEqual(data["count"], 1) + self.assertEqual(data["count"], 2) - self.assertEqual(data["results"][0]["id"], self.job.id) + self.assertEqual({result["id"] for result in data["results"]}, {self.job.id, self.private_job.id}) def test_filter_by_status(self): self.job.status = IndicatorPeriodAggregationJob.Status.FINISHED diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index d3309414e4..bcf89831f1 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -27,7 +27,7 @@ def setUp(self): # Create results framework self.result, self.indicator, self.period = self._make_results_framework(self.project) - self.job = IndicatorPeriodAggregationJob.objects.create(period=self.period, program=self.project) + self.job = IndicatorPeriodAggregationJob.objects.create(period=self.period, root_period=self.period) def _make_project(self, name, public=True): project = self.create_project(f"{name} project", public=public) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index e3b1387e4e..a2d32d7034 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -52,8 +52,8 @@ def schedule_aggregation_job(period: IndicatorPeriod) -> IndicatorPeriodAggregat existing_job.save() return existing_job - program = period.indicator.result.project.ancestor() - return IndicatorPeriodAggregationJob.objects.create(period=period, program=program) + root_period = period.get_root_period() + return IndicatorPeriodAggregationJob.objects.create(period=period, root_period=root_period) def execute_aggregation_jobs(): @@ -140,13 +140,13 @@ def email_job_owners( message=message_template, msg_context={ "indicator": failed_job.period.indicator, - "program": failed_job.program, + "root_project": failed_job.root_project, "reason": reason, } ) def get_job_recipients(job: IndicatorPeriodAggregationJob): - return job.program.primary_organisation.employees.filter( + return job.root_project.primary_organisation.employees.filter( receives_indicator_aggregation_emails=True ).select_related("user") diff --git a/akvo/templates/indicator_aggregation/fail_message.html b/akvo/templates/indicator_aggregation/fail_message.html index 7d5111df30..74bd259ef6 100644 --- a/akvo/templates/indicator_aggregation/fail_message.html +++ b/akvo/templates/indicator_aggregation/fail_message.html @@ -14,8 +14,8 @@ {% blocktrans %} -

The indicator "{{ indicator.title }}" in the hierarchy of the program - {{ program.title }} +

The indicator "{{ indicator.title }}" in the hierarchy of the program/project + {{ root_project.title }} had triggered an aggregation job.

diff --git a/akvo/templates/indicator_aggregation/success_message.html b/akvo/templates/indicator_aggregation/success_message.html index 01d3c28950..394d17cbeb 100644 --- a/akvo/templates/indicator_aggregation/success_message.html +++ b/akvo/templates/indicator_aggregation/success_message.html @@ -14,8 +14,8 @@ {% blocktrans %} -

The indicator: "{{ indicator.title }}" in the hierarchy of the program - {{ program.title }} +

The indicator: "{{ indicator.title }}" in the hierarchy of the program/project + {{ root_project.title }} had triggered a job that had failed.

From baff83f4065e0da0fa5155cb8cc3d88dc98f23b3 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 2 Nov 2022 14:42:33 +0100 Subject: [PATCH 30/59] chore: typos, doc, and test renaming #5145: Feature Request: UI for aggregation tasks --- .../tests/rest/jobs/test_indicator_period_aggregation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py index f8f560d39d..e026fd359d 100644 --- a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py +++ b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py @@ -54,7 +54,7 @@ def setUp(self): self.other_private_job = schedule_aggregation_job(self.other_private_period) def test_super_user(self): - """Super users be able to access all jobs""" + """Super users should be able to access all jobs""" self.c.login(username=self.user.username, password="password") response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json") self.assertEqual(response.status_code, HTTP_200_OK) @@ -88,7 +88,8 @@ def _test_private_user_access(self, expected_job_id_set): self.assertEqual(data.get("count"), len(expected_job_id_set)) self.assertEqual({result["id"] for result in data["results"]}, expected_job_id_set) - def test_filter_by_program(self): + def test_filter_by_root_period(self): + """Ensure that the jobs of the child periods are returned""" self.c.login(username=self.user.username, password="password") response = self.c.get("/rest/v1/jobs/indicator_period_aggregation/?format=json&filter={'root_period_id':%s}" % ( self.period.id @@ -102,6 +103,7 @@ def test_filter_by_program(self): self.assertEqual({result["id"] for result in data["results"]}, {self.job.id, self.private_job.id}) def test_filter_by_status(self): + """Ensure filtering by status works""" self.job.status = IndicatorPeriodAggregationJob.Status.FINISHED self.job.save() @@ -117,6 +119,7 @@ def test_filter_by_status(self): self.assertEqual(data["results"][0]["id"], self.job.id) def test_get_by_id(self): + """Ensure the detail view works as expected""" self.c.login(username=self.user.username, password="password") response = self.c.get(f"/rest/v1/jobs/indicator_period_aggregation/{self.private_job.id}/?format=json") From 3799d6319527c25c045e73eff779babec8f36439 Mon Sep 17 00:00:00 2001 From: Iwan Firmawan Date: Thu, 3 Nov 2022 20:39:17 +0700 Subject: [PATCH 31/59] [#5145] Set job status in each root period Create a global state `programmeRdr` to store program results and update job status in each root period. --- akvo/rsr/spa/app/components/Flex.scss | 2 +- .../rsr/spa/app/images/rsr-alert-triangle.svg | 3 + akvo/rsr/spa/app/images/rsr-loader.svg | 2 +- .../spa/app/modules/program/ActualValue.jsx | 60 ++++---- .../app/modules/program/AggregatedActual.jsx | 137 +++++++++++++++--- .../spa/app/modules/program/Aggregation.jsx | 4 +- .../spa/app/modules/program/ProgramPeriod.jsx | 29 +++- .../app/modules/program/ProjectSummary.jsx | 6 +- akvo/rsr/spa/app/modules/program/config.js | 41 +++++- akvo/rsr/spa/app/modules/program/program.jsx | 18 ++- akvo/rsr/spa/app/modules/program/result.jsx | 22 ++- akvo/rsr/spa/app/modules/program/services.js | 43 ++++-- .../app/modules/program/store/action-types.js | 5 + .../spa/app/modules/program/store/actions.js | 13 ++ .../spa/app/modules/program/store/reducer.js | 61 ++++++++ akvo/rsr/spa/app/modules/program/styles.scss | 45 +++--- akvo/rsr/spa/app/store/config.js | 7 +- akvo/rsr/spa/app/store/root-reducer.js | 4 +- akvo/rsr/spa/app/utils/icons.js | 2 + 19 files changed, 392 insertions(+), 112 deletions(-) create mode 100644 akvo/rsr/spa/app/images/rsr-alert-triangle.svg create mode 100644 akvo/rsr/spa/app/modules/program/store/action-types.js create mode 100644 akvo/rsr/spa/app/modules/program/store/actions.js create mode 100644 akvo/rsr/spa/app/modules/program/store/reducer.js diff --git a/akvo/rsr/spa/app/components/Flex.scss b/akvo/rsr/spa/app/components/Flex.scss index 3a76da3457..71e08d6482 100644 --- a/akvo/rsr/spa/app/components/Flex.scss +++ b/akvo/rsr/spa/app/components/Flex.scss @@ -3,7 +3,7 @@ flex-direction: row; justify-content: flex-end; align-items: center; - & > div.col { + & > div.col:not(:last-child) { margin-right: 9px; } div.col.icon { diff --git a/akvo/rsr/spa/app/images/rsr-alert-triangle.svg b/akvo/rsr/spa/app/images/rsr-alert-triangle.svg new file mode 100644 index 0000000000..f49c542551 --- /dev/null +++ b/akvo/rsr/spa/app/images/rsr-alert-triangle.svg @@ -0,0 +1,3 @@ + + + diff --git a/akvo/rsr/spa/app/images/rsr-loader.svg b/akvo/rsr/spa/app/images/rsr-loader.svg index 22cc17661c..f0ce17bfc2 100644 --- a/akvo/rsr/spa/app/images/rsr-loader.svg +++ b/akvo/rsr/spa/app/images/rsr-loader.svg @@ -1,3 +1,3 @@ - + diff --git a/akvo/rsr/spa/app/modules/program/ActualValue.jsx b/akvo/rsr/spa/app/modules/program/ActualValue.jsx index 71736f466d..cfa5b6c2de 100644 --- a/akvo/rsr/spa/app/modules/program/ActualValue.jsx +++ b/akvo/rsr/spa/app/modules/program/ActualValue.jsx @@ -1,38 +1,48 @@ -import { Spin } from 'antd' -import React, { useState } from 'react' +import React from 'react' +import { Button } from 'antd' import Icon from '../../components/Icon' import Aggregation from './Aggregation' -import { popOver, statusIcons } from './config' -import { getJobStatusByPeriod } from './services' +import { + actualValueIcons, + callToAction, + jobStatus, + toolTips +} from './config' const ActualValue = ({ actualValue, - periodId, + job = {}, }) => { - const [job, setJob] = useState(null) - getJobStatusByPeriod(periodId) - ?.then((res) => { - setJob(res?.shift()) - }) - ?.catch(() => setJob({ status: 'FAILED' })) - const title = popOver[job?.status] ? popOver[job.status]?.title : null - const iconType = statusIcons[job?.status] || null + const _status = (!job?.id && job?.status === jobStatus.maxxed) ? jobStatus.failed : job?.status + const title = toolTips[_status] || null + const iconType = actualValueIcons[_status] || null + + const handleOnRestartJob = () => { + console.log('call to API') + } + return ( - - { - (job === null) - ? } /> - : ( - - - - ) - } - + {_status && ( + + + { + (callToAction.includes(job?.status) && job?.id) + ? ( + + ) + : ( + + ) + } + + + )} - {actualValue || '...'} + {actualValue} diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx index 1fbf637cac..d922225069 100644 --- a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -1,30 +1,133 @@ -import React from 'react' +import React, { useState, useEffect } from 'react' +import { Button, List, Modal, Spin } from 'antd' +import { connect } from 'react-redux' +import chunk from 'lodash/chunk' + import Icon from '../../components/Icon' import Aggregation from './Aggregation' -import { statusIcons } from './config' +import { aggregatedIcons } from './config' +import { getAllJobByRootPeriod } from './services' +import * as actions from './store/actions' const AggregatedActual = ({ value, status, amount, total, - callback, + periodStart, + periodEnd, + periodId, + setRootPeriodJobStatus, + jobs, }) => { - const iconType = statusIcons[status] || statusIcons.FINISHED + const [popUp, setPopUp] = useState(false) + const [page, setPage] = useState(0) + const [history, setHistory] = useState([]) + const [items, setItems] = useState([]) + const [preload, setPreload] = useState({ + fetched: (jobs === undefined), + created: (jobs === undefined), + }) + const pages = chunk(jobs || [], 12) + + const handleOnLoadMore = (_page) => { + setPage(_page) + setHistory([...history, ...pages[_page]]) + } + + const LoadMoreButton = () => { + if (jobs && (history.length < jobs.length)) { + return ( +
+ +
+ ) + } + return null + } + + useEffect(() => { + if (items.length === 0 && preload.fetched && preload.created) { + setPreload({ + ...preload, + fetched: false, + }) + getAllJobByRootPeriod(periodId) + .then((res) => { + if (res.length === 0) { + setPreload({ + created: false, + fetched: false, + }) + } + setItems(res) + }) + .catch(() => { + setPreload({ + created: false, + fetched: false, + }) + }) + } + if (items.length && !preload.fetched && preload.created) { + setPreload({ + ...preload, + created: false, + }) + setRootPeriodJobStatus(periodId, items) + } + if (jobs && history.length === 0) { + setHistory(pages[page]) + } + }, [items, preload, history, jobs]) + + const iconType = aggregatedIcons[status] || null return ( - - - - - - - - - {value} - - - + <> + + + {iconType && ( + setPopUp(!popUp)}> + + + )} + {(preload.fetched || preload.created) && } spinning />} + + + + {value} + + + + setPopUp(!popUp)} + cancelButtonProps={{ style: { display: 'none' } }} + > + + )} + renderItem={item => ( + + } + title={item?.period} + description={item?.updatedAt} + /> + + )} + /> + + ) } -export default AggregatedActual +export default connect( + ({ programmeRdr }) => ({ programmeRdr }), actions +)(AggregatedActual) diff --git a/akvo/rsr/spa/app/modules/program/Aggregation.jsx b/akvo/rsr/spa/app/modules/program/Aggregation.jsx index 48d1630275..590acfe1b4 100644 --- a/akvo/rsr/spa/app/modules/program/Aggregation.jsx +++ b/akvo/rsr/spa/app/modules/program/Aggregation.jsx @@ -22,10 +22,10 @@ const ContentPopOver = ({ status, callback, amount = 2, total = 85 }) => {
{mdOutput(mdParse(description))}
- {(content?.action && callback) && ( + {(callback) && (
)} diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index 11e40d3e29..934aca1adc 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -4,6 +4,8 @@ import { Collapse, Select } from 'antd' import moment from 'moment' import classNames from 'classnames' import { useTranslation } from 'react-i18next' +import groupBy from 'lodash/groupBy' +import uniq from 'lodash/uniq' import countriesDict from '../../utils/countries-dict' import { setNumberFormat } from '../../utils/misc' @@ -16,6 +18,7 @@ import Disaggregations from './Disaggregations' import Icon from '../../components/Icon' import ActualValue from './ActualValue' import AggregatedActual from './AggregatedActual' +import { getSummaryStatus } from './services' const { Panel } = Collapse const { Option } = Select @@ -40,7 +43,7 @@ const ProjectHeader = ({  

- + ) } @@ -63,10 +66,15 @@ const PeriodHeader = ({ periodStart, periodEnd, disaggregationContributions, - disaggregationTargets + disaggregationTargets, + periodId, + jobs, }) => { const { t } = useTranslation() - const handleOnClickIcon = () => console.log('open collapse') + + const groupedStatus = groupBy(jobs || [], 'status') + const allStatus = uniq(Object.keys(groupedStatus)) + const job = getSummaryStatus(allStatus) return ( <>
@@ -91,11 +99,16 @@ const PeriodHeader = ({
aggregated actual
{targetsAt && targetsAt === 'period' && targetValue > 0 && ( @@ -276,7 +289,7 @@ const ProgramPeriod = ({ const approves = subproject.updates.filter(it => it.status && it.status.code === 'A') return (
  • -
    +
    {subproject.projectTitle}

    {subproject.projectSubtitle && {subproject.projectSubtitle}} diff --git a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx index 160a85f848..f0168ce17b 100644 --- a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx +++ b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx @@ -24,7 +24,7 @@ const ProjectSummary = ({ updatesValue, updates, contributors, - periodId, + job }) => { if (indicatorType === 'quantitative') { return ( @@ -39,7 +39,7 @@ const ProjectSummary = ({ openedItem === _index ? (

    - + {actualValue > 0 && {Math.round(((updatesValue) / actualValue) * 100 * 10) / 10}%} {updates.length > 0 &&
    @@ -54,7 +54,7 @@ const ProjectSummary = ({ : (
    - + {aggFilteredTotal > 0 && {Math.round((actualValue / aggFilteredTotal) * 100 * 10) / 10}%}
    ) diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index e0460e4ae1..17fa384313 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -22,12 +22,28 @@ export const sizes = { } } -export const statusIcons = { - RETRY: 'rsr.repeat', +export const jobStatus = { + scheduled: 'SCHEDULED', + running: 'RUNNING', + finished: 'FINISHED', + failed: 'FAILED', + maxxed: 'MAXXED', +} + +export const aggregatedIcons = { + SCHEDULED: 'rsr.clock', + RUNNING: 'rsr.loader', + FINISHED: 'rsr.circle.check', + FAILED: 'rsr.circle.alert', + MAXXED: 'rsr.circle.alert', +} + +export const actualValueIcons = { SCHEDULED: 'rsr.clock', RUNNING: 'rsr.loader', FINISHED: 'rsr.circle.check', FAILED: 'rsr.circle.alert', + MAXXED: 'rsr.repeat', } export const popOver = { @@ -41,11 +57,28 @@ export const popOver = { }, FINISHED: { title: 'Cron Job Finished', - description: 'Aggregated actual is valid', + description: 'Aggregated value is valid', }, FAILED: { title: 'Cron Job Failed', description: '**:value:** out of **:total:** failed to update', action: 'view all' - } + }, + MAXXED: { + title: 'Max attempts reached', + description: '**:value:** out of **:total:** max attempts reached', + action: 'view all' + }, } + +export const toolTips = { + SCHEDULED: popOver.SCHEDULED.title, + RUNNING: popOver.RUNNING.title, + FINISHED: popOver.FINISHED.title, + FAILED: popOver.FAILED.title, + MAXXED: 'Restart the job', +} + +export const callToAction = [ + jobStatus.maxxed, +] diff --git a/akvo/rsr/spa/app/modules/program/program.jsx b/akvo/rsr/spa/app/modules/program/program.jsx index e2f30c815b..8cee2ef8e9 100644 --- a/akvo/rsr/spa/app/modules/program/program.jsx +++ b/akvo/rsr/spa/app/modules/program/program.jsx @@ -13,7 +13,8 @@ import api from '../../utils/api' import Reports from '../reports/reports' import countriesDict from '../../utils/countries-dict' import StickyClass from './sticky-class' -import * as actions from '../editor/actions' +import { setProjectTitle } from '../editor/actions' +import * as programmeActions from './store/actions' const { Panel } = Collapse const { TabPane } = Tabs @@ -25,9 +26,13 @@ const ExpandIcon = ({ isActive }) => (
    ) -const Program = ({ match: {params}, userRdr, ...props }) => { +const Program = ({ + match: { params }, + userRdr, + programmeRdr: results, + ...props +}) => { const { t } = useTranslation() - const [results, setResults] = useState([]) const [title, setTitle] = useState('') const [loading, setLoading] = useState(true) const [countryOpts, setCountryOpts] = useState([]) @@ -38,7 +43,8 @@ const Program = ({ match: {params}, userRdr, ...props }) => { if (params.projectId !== 'new') { api.get(`/project/${params.projectId}/results`) .then(({ data }) => { - setResults(data.results.map(it => ({ ...it, indicators: [], fetched: false }))) + const _results = data.results.map(it => ({ ...it, indicators: [], fetched: false })) + props.setProgrammeResults(_results) setTitle(data.title) props.setProjectTitle(data.title) document.title = `${data.title} | Akvo RSR` @@ -111,7 +117,7 @@ const Program = ({ match: {params}, userRdr, ...props }) => { )} > - + )} @@ -136,5 +142,5 @@ const Program = ({ match: {params}, userRdr, ...props }) => { } export default connect( - ({ editorRdr: {section1: {fields: {title}}}, userRdr }) => ({ title, userRdr }), actions + ({ editorRdr: {section1: {fields: {title}}}, userRdr, programmeRdr }) => ({ title, userRdr, programmeRdr }), ({ ...programmeActions, setProjectTitle }) )(Program) diff --git a/akvo/rsr/spa/app/modules/program/result.jsx b/akvo/rsr/spa/app/modules/program/result.jsx index 6195cc4efa..56f942ae5d 100644 --- a/akvo/rsr/spa/app/modules/program/result.jsx +++ b/akvo/rsr/spa/app/modules/program/result.jsx @@ -2,9 +2,11 @@ import React, { useEffect } from 'react' import { Collapse, Icon, Spin } from 'antd' import classNames from 'classnames' import { useTranslation } from 'react-i18next' +import { connect } from 'react-redux' import Indicator from './indicator' import api from '../../utils/api' import StickyClass from './sticky-class' +import * as actions from './store/actions' const { Panel } = Collapse const ExpandIcon = ({ isActive }) => ( @@ -21,8 +23,8 @@ const Result = ({ indicators, targetsAt, fetched, - results, - setResults, + programmeRdr: results, + ...props }) => { const { t } = useTranslation() useEffect(() => { @@ -32,19 +34,11 @@ const Result = ({ ?.get(`/project/${programId}/result/${id}/`) ?.then(({ data }) => { if (resultIndex > -1) { - setResults([ - ...results.slice(0, resultIndex), - { ...results[resultIndex], indicators: data.indicators, fetched: true }, - ...results.slice(resultIndex + 1) - ]) + props.updateProgrammePerResult(resultIndex, { ...data, fetched: true }) } }) ?.catch(() => { - setResults([ - ...results.slice(0, resultIndex), - { ...results[resultIndex], fetched: true }, - ...results.slice(resultIndex + 1) - ]) + props.updateProgrammePerResult(resultIndex, { fetched: true }) }) } }, [fetched, indicators]) @@ -72,4 +66,6 @@ const Result = ({ ) } -export default Result +export default connect( + ({ programmeRdr }) => ({ programmeRdr }), actions +)(Result) diff --git a/akvo/rsr/spa/app/modules/program/services.js b/akvo/rsr/spa/app/modules/program/services.js index 645eb8a4fb..5d545a3ddf 100644 --- a/akvo/rsr/spa/app/modules/program/services.js +++ b/akvo/rsr/spa/app/modules/program/services.js @@ -1,20 +1,41 @@ import api from '../../utils/api' +import { jobStatus } from './config' -export const getAggregationJobsApi = async (id, page = 1) => { - /** - * Choices are: attempts, id, period, period_id, pid, program, program_id, status, updated_at - * ?format=json&filter={'program_id':9062} - */ - const response = await api.get(`/jobs/indicator_period_aggregation/?page=${page}&program_id=${id}&format=json`) +/** + * Choices are: + * attempts, id, period, period_id, pid, root_period, root_period_id, status, updated_at + * */ + +export const getJobStatusByPeriod = async (period) => { + const response = await api.get(`/jobs/indicator_period_aggregation/?format=json&filter={'period': ${period}}`) + const { results } = response.data + return results +} + +export const getJobStatusByRootPeriod = async (rootPeriod) => { + const response = await api.get(`/jobs/indicator_period_aggregation/?format=json&filter={'root_period':${rootPeriod}}`) + const { results } = response?.data + return results +} + +export const getSummaryStatus = allStatus => { + const isJobFailed = allStatus?.filter((a) => [jobStatus.failed, jobStatus.maxxed]?.includes(a)) + const highlighted = allStatus?.length > 1 ? allStatus?.filter((a) => a !== jobStatus.finished) : allStatus + const _status = isJobFailed.length ? isJobFailed.pop() : highlighted.pop() || null + return ({ status: _status }) +} + +export const getAllJobByRootPeriod = async (rootPeriod, page = 1) => { + const response = await api.get(`/jobs/indicator_period_aggregation/?format=json&filter={'root_period':${rootPeriod}}&page=${page}`) const { results, next } = response.data if (next) { - return results?.concat(await getAggregationJobsApi(id, page + 1)) + return results?.concat(await getAllJobByRootPeriod(rootPeriod, page + 1)) } return results } -export const getJobStatusByPeriod = async (period) => { - const response = await api.get(`/jobs/indicator_period_aggregation/?format=json&filter={'period': ${period}}`) - const { results } = response.data - return results +export const getAllResponse = async (responses, callback) => { + let data = await Promise.all(responses) + data = data?.flatMap((d) => d) + if (callback) callback(data) } diff --git a/akvo/rsr/spa/app/modules/program/store/action-types.js b/akvo/rsr/spa/app/modules/program/store/action-types.js new file mode 100644 index 0000000000..3d22b93d3f --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/store/action-types.js @@ -0,0 +1,5 @@ +export default { + APPEND_RESULTS: 'PG_APPEND_RESULTS', + UPDATE_RESULT: 'PG_UPDATE_RESULT', + SET_JOB_STATUS: 'PG_SET_JOB_STATUS', +} diff --git a/akvo/rsr/spa/app/modules/program/store/actions.js b/akvo/rsr/spa/app/modules/program/store/actions.js new file mode 100644 index 0000000000..ac4dc53d60 --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/store/actions.js @@ -0,0 +1,13 @@ +import actionTypes from './action-types' + +export const setProgrammeResults = (payload) => (dispatch) => { + dispatch({ type: actionTypes.APPEND_RESULTS, payload }) +} + +export const updateProgrammePerResult = (resultIndex, data) => (dispatch) => { + dispatch({ type: actionTypes.UPDATE_RESULT, payload: { resultIndex, data } }) +} + +export const setRootPeriodJobStatus = (rootPeriod, results) => (dispatch) => { + dispatch({ type: actionTypes.SET_JOB_STATUS, payload: { rootPeriod, results } }) +} diff --git a/akvo/rsr/spa/app/modules/program/store/reducer.js b/akvo/rsr/spa/app/modules/program/store/reducer.js new file mode 100644 index 0000000000..0d178657b8 --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/store/reducer.js @@ -0,0 +1,61 @@ +import uniq from 'lodash/uniq' + +import actionTypes from './action-types' +import { getSummaryStatus } from '../services' + +export default (state = [], action) => { + switch (action.type) { + case actionTypes.APPEND_RESULTS: + return action.payload + case actionTypes.UPDATE_RESULT: + const { resultIndex, data } = action.payload + return [ + ...state.slice(0, resultIndex), + { ...state[resultIndex], ...data }, + ...state.slice(resultIndex + 1) + ] + case actionTypes.SET_JOB_STATUS: + const { rootPeriod, results } = action.payload + return state?.map((s) => ({ + ...s, + indicators: s?.indicators?.map((i) => ({ + ...i, + periods: i?.periods?.map((p) => { + if (p?.periodId === rootPeriod) { + const _contributors = p?.contributors?.map((cb) => { + // parent contributor + const parentJobs = results?.filter((rs) => rs?.period === cb?.periodId) + const _subContributors = cb?.contributors?.map((subCb) => { + // child contributor + const jobs = results?.filter((rs) => rs?.period === subCb?.periodId) + const firstJob = jobs?.shift() || {} // assuming the latest update is in first order. + return ({ + ...subCb, + job: firstJob + }) + }) + const allStatus = uniq(_subContributors?.map((subC) => subC?.job?.status))?.filter((status) => status) + const job = parentJobs?.length + ? parentJobs.shift() + : getSummaryStatus(allStatus) + return ({ + ...cb, + job, + contributors: _subContributors + }) + }) + const jobs = results?.filter((r) => (r?.status)) + return ({ + ...p, + jobs, + contributors: _contributors + }) + } + return p + }) + })) + })) + default: + return state + } +} diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index 3139ccd8ab..2ff02ff6c1 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -356,7 +356,6 @@ span{ text-transform: uppercase; font-size: 11px; - color: #333; } } } @@ -427,7 +426,7 @@ font-weight: 300; } .d-flex { - .icon .FAILED { + .icon .FAILED, .icon .MAXXED { color: $error700; } b { @@ -622,14 +621,6 @@ display: block; } } - .d-flex .col.icon { - &> .FAILED { - color: $error700; - } - &> .FINISHED { - color: $primary700; - } - } } position: relative; .total{ @@ -740,6 +731,9 @@ border-top: none; .sub-contributors{ margin: 0 13px; + .max-w-1180 { + max-width: 1180px; + } } } } @@ -823,6 +817,8 @@ } small{ white-space: nowrap; + width: 100%; + text-align: right; } position: relative; &:hover{ @@ -830,14 +826,6 @@ display: block; } } - .d-flex .col.icon { - &> .FAILED { - color: $error700; - } - &> .FINISHED { - color: $primary700; - } - } } .updates{ width: 100%; @@ -1138,7 +1126,7 @@ .ant-popover { max-width: 206px; - &.FAILED .ant-popover-inner-content{ + &.FAILED .ant-popover-inner-content, &.MAXXED .ant-popover-inner-content { background-color: $error50; } .description { @@ -1151,3 +1139,22 @@ text-transform: capitalize; } } + +.d-flex .col.icon { + &> .FAILED, .MAXXED { + color: $error700; + } + &> .FINISHED { + color: $primary700; + } + .ant-btn.ant-btn-circle .MAXXED { + padding-top: 4px; + } +} + +.load-more-container { + text-align: center; + margin-top: 12px; + height: 32px; + line-height: 32px; +} diff --git a/akvo/rsr/spa/app/store/config.js b/akvo/rsr/spa/app/store/config.js index 7af346d413..9a1a8b2d47 100644 --- a/akvo/rsr/spa/app/store/config.js +++ b/akvo/rsr/spa/app/store/config.js @@ -10,7 +10,12 @@ export default (initialState) => { const rootPersistConfig = { key: 'root', storage, - blacklist: ['userRdr', 'editorRdr', 'resultRdr'] + blacklist: [ + 'userRdr', + 'editorRdr', + 'resultRdr', + 'programmeRdr', + ] } const persistedReducer = persistReducer(rootPersistConfig, rootReducer) const store = createStore( diff --git a/akvo/rsr/spa/app/store/root-reducer.js b/akvo/rsr/spa/app/store/root-reducer.js index 7ed7e8a0ff..f18b248410 100644 --- a/akvo/rsr/spa/app/store/root-reducer.js +++ b/akvo/rsr/spa/app/store/root-reducer.js @@ -5,6 +5,7 @@ import storage from 'redux-persist/lib/storage' import editorRdr from '../modules/editor/reducer' import userRdr from './user-reducer' import resultRdr from '../modules/results/reducer' +import programmeRdr from '../modules/program/store/reducer' const userPersistConfig = { key: 'userRdr', @@ -29,7 +30,8 @@ const editorPersistConfig = { const rootReducer = combineReducers({ editorRdr: persistReducer(editorPersistConfig, editorRdr), userRdr: persistReducer(userPersistConfig, userRdr), - resultRdr + resultRdr, + programmeRdr, }) export default rootReducer diff --git a/akvo/rsr/spa/app/utils/icons.js b/akvo/rsr/spa/app/utils/icons.js index 3fd582f0a0..1e31856b25 100644 --- a/akvo/rsr/spa/app/utils/icons.js +++ b/akvo/rsr/spa/app/utils/icons.js @@ -3,6 +3,7 @@ import rsrCheckCircle from '../images/rsr-check-circle.svg' import rsrRepeat from '../images/rsr-repeat.svg' import rsrLoader from '../images/rsr-loader.svg' import rsrClock from '../images/rsr-clock.svg' +import rsrWarning from '../images/rsr-alert-triangle.svg' export const icons = { rsr: { @@ -13,5 +14,6 @@ export const icons = { repeat: rsrRepeat, loader: rsrLoader, clock: rsrClock, + warning: rsrWarning, } } From 0cc76c9c51b19b7fd774d35315db001e1cc5c1f1 Mon Sep 17 00:00:00 2001 From: Iwan Firmawan Date: Tue, 8 Nov 2022 13:49:19 +0700 Subject: [PATCH 32/59] [#5145] Create a cron job history pop-up --- .../app/modules/program/AggregatedActual.jsx | 81 ++++++++++++++++--- .../spa/app/modules/program/Aggregation.jsx | 2 +- .../spa/app/modules/program/ProgramPeriod.jsx | 3 +- akvo/rsr/spa/app/modules/program/config.js | 8 ++ akvo/rsr/spa/app/modules/program/styles.scss | 16 ++++ akvo/rsr/spa/app/utils/dates.js | 4 + 6 files changed, 102 insertions(+), 12 deletions(-) create mode 100644 akvo/rsr/spa/app/utils/dates.js diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx index d922225069..43bbfdc688 100644 --- a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -1,13 +1,27 @@ import React, { useState, useEffect } from 'react' -import { Button, List, Modal, Spin } from 'antd' +import { + Button, + Row, + Col, + List, + Modal, + Spin, + Tag, + Tooltip, + Typography, +} from 'antd' import { connect } from 'react-redux' import chunk from 'lodash/chunk' +import moment from 'moment' import Icon from '../../components/Icon' import Aggregation from './Aggregation' -import { aggregatedIcons } from './config' +import { aggregatedIcons, jobStatusColor, toolTips } from './config' import { getAllJobByRootPeriod } from './services' import * as actions from './store/actions' +import { printIndicatorPeriod } from '../../utils/dates' + +const { Text, Title } = Typography const AggregatedActual = ({ value, @@ -17,6 +31,7 @@ const AggregatedActual = ({ periodStart, periodEnd, periodId, + programmeRdr, setRootPeriodJobStatus, jobs, }) => { @@ -29,6 +44,20 @@ const AggregatedActual = ({ created: (jobs === undefined), }) const pages = chunk(jobs || [], 12) + const _periods = programmeRdr + ?.flatMap((r) => r?.indicators) + ?.flatMap((i) => i?.periods?.map((p) => ({ ...p, indicator: { id: i?.id, title: i?.title } }))) + + const getProjectByPeriodID = (ID) => { + const _contrib = _periods?.flatMap((p) => p?.contributors) + const _contributors = [ + ..._contrib, + ..._contrib?.flatMap((cb) => cb?.contributors) + ] + return _contributors?.find((cb) => cb?.periodId === ID) + } + + const getIndicatorByPeriodID = (ID) => _periods?.find((p) => p?.periodId === ID) const handleOnLoadMore = (_page) => { setPage(_page) @@ -84,6 +113,7 @@ const AggregatedActual = ({ }, [items, preload, history, jobs]) const iconType = aggregatedIcons[status] || null + const dataIndicator = getIndicatorByPeriodID(periodId) return ( <> @@ -106,22 +136,53 @@ const AggregatedActual = ({ visible={popUp} onOk={() => setPopUp(!popUp)} cancelButtonProps={{ style: { display: 'none' } }} + title="Cron Job History" + width={650} > +
    + Indicator + + {dataIndicator?.indicator?.title || ''} + + Period + + {printIndicatorPeriod(periodStart, periodEnd)} + +
    )} - renderItem={item => ( - - } - title={item?.period} - description={item?.updatedAt} - /> - + header={( + Details )} + renderItem={item => { + const _project = getProjectByPeriodID(item?.period) + return ( + + + + + )} + title={_project?.projectTitle} + description={( + + + {item?.status} + + + {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} + + + )} + /> + + ) + }} /> diff --git a/akvo/rsr/spa/app/modules/program/Aggregation.jsx b/akvo/rsr/spa/app/modules/program/Aggregation.jsx index 590acfe1b4..b76a6cd5c1 100644 --- a/akvo/rsr/spa/app/modules/program/Aggregation.jsx +++ b/akvo/rsr/spa/app/modules/program/Aggregation.jsx @@ -18,7 +18,7 @@ const ContentPopOver = ({ status, callback, amount = 2, total = 85 }) => { const mdOutput = SimpleMarkdown.defaultOutput return ( <> -

    {content?.title}

    +

    {content?.title}

    {mdOutput(mdParse(description))}
    diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index 934aca1adc..0cc2fc2b32 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -19,6 +19,7 @@ import Icon from '../../components/Icon' import ActualValue from './ActualValue' import AggregatedActual from './AggregatedActual' import { getSummaryStatus } from './services' +import { printIndicatorPeriod } from '../../utils/dates' const { Panel } = Collapse const { Option } = Select @@ -78,7 +79,7 @@ const PeriodHeader = ({ return ( <>
    -
    {moment(periodStart, 'DD/MM/YYYY').format('DD MMM YYYY')} - {moment(periodEnd, 'DD/MM/YYYY').format('DD MMM YYYY')}
    +
    {printIndicatorPeriod(periodStart, periodEnd)}
    • {filteredContributors.length} {t('contributor_s', { count: filteredContributors.length })}
    • {filteredCountries.length} {t('country_s', { count: filteredCountries.length })}
      • diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index 17fa384313..cf25c0e4e5 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -46,6 +46,14 @@ export const actualValueIcons = { MAXXED: 'rsr.repeat', } +export const jobStatusColor = { + SCHEDULED: '#667085', + RUNNING: 'gold', + FINISHED: 'blue', + FAILED: 'magenta', + MAXXED: 'red', +} + export const popOver = { SCHEDULED: { title: 'Cron Job Scheduled', diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index 2ff02ff6c1..fa6fd5ab3d 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -1152,6 +1152,22 @@ } } +.text-right { + text-align: right; +} +.SVGInline { + &.FAILED, &.MAXXED { + color: $error700; + } + &.FINISHED { + color: $primary700; + } +} +.modal-descriptions { + h4.ant-typography { + margin: .2em 0; + } +} .load-more-container { text-align: center; margin-top: 12px; diff --git a/akvo/rsr/spa/app/utils/dates.js b/akvo/rsr/spa/app/utils/dates.js new file mode 100644 index 0000000000..70bf450afd --- /dev/null +++ b/akvo/rsr/spa/app/utils/dates.js @@ -0,0 +1,4 @@ +import moment from 'moment' + +export const printIndicatorPeriod = (periodStart, periodEnd) => + `${moment(periodStart, 'DD/MM/YYYY').format('DD MMM YYYY')} - ${moment(periodEnd, 'DD/MM/YYYY').format('DD MMM YYYY')}` From 9f6de5d8ae1043b260daf0df40aa850d703b7c46 Mon Sep 17 00:00:00 2001 From: Iwan Firmawan Date: Tue, 8 Nov 2022 14:41:11 +0700 Subject: [PATCH 33/59] [#5145] Sort Jobs by `updatedAt` descending --- akvo/rsr/spa/app/modules/program/store/reducer.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/spa/app/modules/program/store/reducer.js b/akvo/rsr/spa/app/modules/program/store/reducer.js index 0d178657b8..c5d4d6a2c5 100644 --- a/akvo/rsr/spa/app/modules/program/store/reducer.js +++ b/akvo/rsr/spa/app/modules/program/store/reducer.js @@ -1,4 +1,5 @@ import uniq from 'lodash/uniq' +import orderBy from 'lodash/orderBy' import actionTypes from './action-types' import { getSummaryStatus } from '../services' @@ -44,7 +45,7 @@ export default (state = [], action) => { contributors: _subContributors }) }) - const jobs = results?.filter((r) => (r?.status)) + const jobs = orderBy(results?.filter((r) => (r?.status)), ['updatedAt'], ['desc']) return ({ ...p, jobs, From 89e2da240aaa467b31265d52f55298aeb50e2a5c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 8 Nov 2022 11:09:04 +0100 Subject: [PATCH 34/59] fix: IndicatorPeriod.get_root_period() not available in migrations The function was copy-pasted from the model class for the migration because apps.get_model() can't make model methods available. #5145: Feature Request: UI for aggregation tasks --- .../0222_indicatorperiodaggregationjob_root_period.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py b/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py index 99437c767d..cb2f80fd88 100644 --- a/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py +++ b/akvo/rsr/migrations/0222_indicatorperiodaggregationjob_root_period.py @@ -4,11 +4,18 @@ import django.db.models.deletion +def get_root_period(period): + root = period + while root.parent_period: + root = root.parent_period + return root + + def set_root_period(apps, schema_editor): IndicatorPeriodAggregationJob = apps.get_model("rsr", "IndicatorPeriodAggregationJob") jobs = [] for job in IndicatorPeriodAggregationJob.objects.all(): - job.root_period = job.period.get_root_period() + job.root_period = get_root_period(job.period) jobs.append(job) IndicatorPeriodAggregationJob.objects.bulk_update(jobs, ["root_period"]) From 8739d05f07ad0d8392f5339bb26839dc2bdc1d81 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 8 Nov 2022 12:04:22 +0100 Subject: [PATCH 35/59] fix: IndicatorPeriodAggregationJob root_period readonly in admin Without that, it tries to list all possible periods and that takes forever. #5145: Feature Request: UI for aggregation tasks --- akvo/rsr/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akvo/rsr/admin.py b/akvo/rsr/admin.py index faef89ddd9..73f3a744a1 100644 --- a/akvo/rsr/admin.py +++ b/akvo/rsr/admin.py @@ -949,7 +949,7 @@ class IndicatorPeriodAggregationJobAdmin(admin.ModelAdmin): list_display = ('indicator_title', 'status', 'project_title', 'root_project_title', 'period', 'updated_at') list_filter = ('status', ) search_fields = ('period__indicator__result__period__title', 'period__indicator__title') - readonly_fields = ('updated_at', 'period', 'project_title', 'root_project_title', 'indicator_title') + readonly_fields = ('updated_at', 'period', 'root_period', 'project_title', 'root_project_title', 'indicator_title') @admin.display(description='Project Title') def project_title(self, obj): From 98e758a0a0cb982b596f0a724790bc8c6790cb59 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 8 Nov 2022 16:30:29 +0100 Subject: [PATCH 36/59] fix: Keep periods in result framework that have an aggregation job Is some are hidden, it's not possible to present the aggregation tasks thereof. #5145: Feature Request: UI for aggregation tasks --- akvo/rest/views/project_overview.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/akvo/rest/views/project_overview.py b/akvo/rest/views/project_overview.py index 268f6f0a91..5235d7c2ff 100644 --- a/akvo/rest/views/project_overview.py +++ b/akvo/rest/views/project_overview.py @@ -15,7 +15,7 @@ from datetime import date from decimal import Decimal, InvalidOperation from django.conf import settings -from django.db.models import Sum +from django.db.models import Count, Sum from django.shortcuts import get_object_or_404 from functools import cached_property, lru_cache from rest_framework.authentication import SessionAuthentication @@ -465,6 +465,9 @@ def _get_indicator_periods_hierarchy_flatlist(indicator): 'disaggregation_targets', 'disaggregation_targets__dimension_value', 'disaggregation_targets__dimension_value__name' + ).annotate( + num_aggregation_jobs=Count("aggregation_jobs"), + num_child_aggregation_jobs=Count("child_aggregation_jobs"), ).filter(pk__in=family) return periods @@ -500,7 +503,11 @@ def _transform_period_contributions_node(node, aggregate_targets=False): is_qualitative = period.indicator.type == QUALITATIVE actual_numerator, actual_denominator = None, None updates_value, updates_numerator, updates_denominator = None, None, None - contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(node['children'], is_percentage) + contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy( + node['children'], + is_percentage, + node['item'].num_child_aggregation_jobs, + ) aggregated_value, aggregated_numerator, aggregated_denominator = aggregates updates = _transform_updates(period) @@ -558,7 +565,7 @@ def _aggregate_targets(node): return aggregate -def _transform_contributions_hierarchy(tree, is_percentage): +def _transform_contributions_hierarchy(tree, is_percentage, root_has_aggregation_job): contributors = [] contributor_countries = [] aggregated_value = Decimal(0) if not is_percentage else None @@ -566,7 +573,7 @@ def _transform_contributions_hierarchy(tree, is_percentage): aggregated_denominator = Decimal(0) if is_percentage else None disaggregations = {} for node in tree: - contributor, countries = _transform_contributor_node(node, is_percentage) + contributor, countries = _transform_contributor_node(node, is_percentage, root_has_aggregation_job) if contributor: contributors.append(contributor) contributor_countries = _merge_unique(contributor_countries, countries) @@ -616,8 +623,8 @@ def _extract_percentage_updates(updates): return numerator, denominator -def _transform_contributor_node(node, is_percentage): - contributor, aggregate_children = _transform_contributor(node['item'], is_percentage) +def _transform_contributor_node(node, is_percentage, root_has_aggregation_job): + contributor, aggregate_children = _transform_contributor(node['item'], is_percentage, root_has_aggregation_job) if not contributor: return contributor, [] @@ -633,7 +640,9 @@ def _transform_contributor_node(node, is_percentage): if not aggregate_children: return contributor, contributor_countries - contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy(node['children'], is_percentage) + contributors, countries, aggregates, disaggregations = _transform_contributions_hierarchy( + node['children'], is_percentage, node['item'].num_child_aggregation_jobs or root_has_aggregation_job + ) aggregated_value, aggregated_numerator, aggregated_denominator = aggregates contributors_count = len(contributors) if contributors_count: @@ -657,14 +666,14 @@ def _calculate_update_values(updates): return total -def _transform_contributor(period, is_percentage): +def _transform_contributor(period, is_percentage, root_has_aggregation_job): value = _force_decimal(period.actual_value) is_qualitative = period.indicator.type == QUALITATIVE # FIXME: Not sure why the value < 1 check is being used, if it is a float # comparison issue, we need to resolve it in a better fashion. # Return early if there are not updates and value is "0" for quantitative updates - if not is_qualitative and value < 1 and period.data.count() < 1: + if not root_has_aggregation_job and not is_qualitative and value < 1 and period.data.count() < 1: return None, None project = period.indicator.result.project From 77dab6d56f104b722f107d2d5afa8ceff3c6799d Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Wed, 9 Nov 2022 11:07:01 +0700 Subject: [PATCH 37/59] [#5145] Fix only get first cron job and remove red color --- akvo/rsr/spa/app/modules/program/store/reducer.js | 6 +++--- akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx | 4 ++-- .../rsr/spa/app/modules/results-overview/ResultOverview.jsx | 2 +- akvo/rsr/spa/app/utils/common.scss | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/store/reducer.js b/akvo/rsr/spa/app/modules/program/store/reducer.js index c5d4d6a2c5..e06f3cbfcf 100644 --- a/akvo/rsr/spa/app/modules/program/store/reducer.js +++ b/akvo/rsr/spa/app/modules/program/store/reducer.js @@ -29,15 +29,15 @@ export default (state = [], action) => { const _subContributors = cb?.contributors?.map((subCb) => { // child contributor const jobs = results?.filter((rs) => rs?.period === subCb?.periodId) - const firstJob = jobs?.shift() || {} // assuming the latest update is in first order. + const latestJob = jobs?.pop() || {} return ({ ...subCb, - job: firstJob + job: latestJob }) }) const allStatus = uniq(_subContributors?.map((subC) => subC?.job?.status))?.filter((status) => status) const job = parentJobs?.length - ? parentJobs.shift() + ? parentJobs.pop() : getSummaryStatus(allStatus) return ({ ...cb, diff --git a/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx b/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx index 8bcf3ecc45..c3b216f17b 100644 --- a/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx @@ -16,7 +16,7 @@ import SimpleMarkdown from 'simple-markdown' import SVGInline from 'react-svg-inline' import classNames from 'classnames' import moment from 'moment' -import { isEmpty } from 'lodash' +import { isEmpty, orderBy } from 'lodash' import { connect } from 'react-redux' import './TobeReported.scss' @@ -152,7 +152,7 @@ const TobeReported = ({ grid={{ column: 1 }} itemLayout="vertical" className="tobe-reported" - dataSource={updates} + dataSource={orderBy(updates, ['indicator.title'], ['asc'])} renderItem={(item, ix) => { const iKey = item?.id || `${item?.indicator?.id}0${ix}` const updateClass = item?.statusDisplay?.toLowerCase()?.replace(/\s+/g, '-') diff --git a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx index 62cbf3a450..29b7cc20b9 100644 --- a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx +++ b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx @@ -3,7 +3,7 @@ import React, { useState } from 'react' import { connect } from 'react-redux' import { Icon, Collapse, notification, Typography, Row, Col } from 'antd' import { useTranslation } from 'react-i18next' -import { cloneDeep, isEmpty } from 'lodash' +import { cloneDeep, isEmpty, orderBy } from 'lodash' import classNames from 'classnames' import SimpleMarkdown from 'simple-markdown' diff --git a/akvo/rsr/spa/app/utils/common.scss b/akvo/rsr/spa/app/utils/common.scss index 8fa65dd456..a5db8760ae 100644 --- a/akvo/rsr/spa/app/utils/common.scss +++ b/akvo/rsr/spa/app/utils/common.scss @@ -1,9 +1,9 @@ $primaryColor: #43998f; -$colors: #A278B5, rgb(182, 74, 106), #DB3056, #FF6464, #FFB99A, #CFB495, #8BBABB, #6C7B95, #32AFA9, #A4D4AE, #698474, #889E81, #839b60, #F8B195, #F67280, #C06C84, #6C5B7B, #6C7261, #E6A157, #EB8242, #C9753D, #e9a5cf, #ca80b1, #F0CF85, #381460, #933B5B, #FF6F5E, #40BFC1, #9ACEFF, #12CAD6, #0FABBC, #151965, #32407B, #515585, #46B5D1, #E9EA77, #FFD369, #a99260, #c38a6d, #d8d13e; +$colors: #560764, #905E96, #D58BDD, #FF99D7, #FFB99A, #CFB495, #8BBABB, #6C7B95, #32AFA9, #A4D4AE, #698474, #889E81, #839b60, #F8B195, #F67280, #C06C84, #6C5B7B, #6C7261, #E6A157, #EB8242, #C9753D, #e9a5cf, #ca80b1, #F0CF85, #381460, #933B5B, #FF6F5E, #40BFC1, #9ACEFF, #12CAD6, #0FABBC, #151965, #32407B, #515585, #46B5D1, #E9EA77, #FFD369, #a99260, #c38a6d, #d8d13e; $g1colors: #8E936D, #598381, #177E89, #08605F, #75894a, #5C80BC, #4D5061, #30323D, #E8C547, #A2AD59; $g2colors: #0c7b93, #00a8cc, #ddba01, #b0a160, #434e52, #5b8c85, #8d6e63, #ea9085, #92cad3, #6d9499; $g3colors: #2c498b, #35619b, #3e78ab, #4891bb, #52aacb, #6abdd0, #8ecccc, #b4dbcb, #8ca26f; -$g4colors: #B40815, #CC0516, #EC1817, #F9532F, #FF9B58, #D62A0B; +$g4colors: #393E46, #6D9886, #F2E7D5, #062C30, #FF9B58, #05595B; $submitted: rgb(24, 144, 255); $error50: #FEF3F2; $error700: #B42318; From 2771dae34108627babbc19c263b676d001629c14 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Wed, 9 Nov 2022 18:31:57 +0700 Subject: [PATCH 38/59] [#5145] Fix collapsible periods and highlight failed job --- .../spa/app/modules/program/ProgramPeriod.jsx | 31 ++++++++++++++----- .../rsr/spa/app/modules/program/indicator.jsx | 9 +++++- akvo/rsr/spa/app/modules/program/styles.scss | 13 +++++++- .../results-overview/ResultOverview.jsx | 2 +- akvo/rsr/spa/app/utils/common.scss | 2 +- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index 0cc2fc2b32..e55712bd47 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -1,6 +1,6 @@ /* global document */ import React from 'react' -import { Collapse, Select } from 'antd' +import { Collapse, Row, Col, Select } from 'antd' import moment from 'moment' import classNames from 'classnames' import { useTranslation } from 'react-i18next' @@ -70,15 +70,26 @@ const PeriodHeader = ({ disaggregationTargets, periodId, jobs, + activePeriods, + setActivePeriods, + periodIndex, }) => { const { t } = useTranslation() + const handleOnClick = () => { + if (activePeriods?.includes(periodIndex)) { + setActivePeriods(activePeriods?.filter((a) => a !== periodIndex)) + } else { + setActivePeriods([...activePeriods, periodIndex]) + } + } + const groupedStatus = groupBy(jobs || [], 'status') const allStatus = uniq(Object.keys(groupedStatus)) const job = getSummaryStatus(allStatus) return ( - <> -
      +
      event.stopPropagation()}> +
      {printIndicatorPeriod(periodStart, periodEnd)}
      • {filteredContributors.length} {t('contributor_s', { count: filteredContributors.length })}
        • @@ -141,7 +152,7 @@ const PeriodHeader = ({
      ) } - +
      ) } @@ -165,6 +176,8 @@ const ProgramPeriod = ({ aggFilteredTotalTarget, aggFilteredTotal, openedItem, + activePeriods, + setActivePeriods, handleAccordionChange, ...props }) => { @@ -222,7 +235,10 @@ const ProgramPeriod = ({ hasDisaggregations, clickBar, mouseEnterBar, - mouseLeaveBar + mouseLeaveBar, + activePeriods, + setActivePeriods, + periodIndex, }} /> )} @@ -267,7 +283,8 @@ const ProgramPeriod = ({ return ( { const approves = subproject.updates.filter(it => it.status && it.status.code === 'A') return ( -
    • +
    • {subproject.projectTitle}

      diff --git a/akvo/rsr/spa/app/modules/program/indicator.jsx b/akvo/rsr/spa/app/modules/program/indicator.jsx index 13429926e6..62e3b00f64 100644 --- a/akvo/rsr/spa/app/modules/program/indicator.jsx +++ b/akvo/rsr/spa/app/modules/program/indicator.jsx @@ -18,6 +18,7 @@ const Indicator = ({ const [pinned, setPinned] = useState(-1) const [openedItem, setOpenedItem] = useState(null) const [countriesFilter, setCountriesFilter] = useState([]) + const [activePeriods, setActivePeriods] = useState([]) const listRef = useRef(null) const pinnedRef = useRef(-1) const tooltipRef = useRef(null) @@ -86,7 +87,11 @@ const Indicator = ({ )} - }> + } + > {periods.map((period, index) => { const filteredContributors = period.contributors.filter(filterProjects) const filteredCountries = countryFilter.length > 0 ? countryFilter : period.countries @@ -116,6 +121,8 @@ const Indicator = ({ countriesFilter, openedItem, periodIndex: index, + activePeriods, + setActivePeriods, handleCountryFilter: setCountriesFilter, handleAccordionChange }} diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index fa6fd5ab3d..92ba0caf85 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -545,7 +545,6 @@ border-color: nth($colors, $i); ul.sub-contributors{ &>li{ - background-color: rgba(nth($colors, $i), 0.15); .value{ b, small{ color: darken(nth($colors, $i), 23%); @@ -738,6 +737,9 @@ } } &.quantitative{ + &.FAILED, &.MAXXED { + border-left: 4px solid $error700; + } .ant-collapse-content ul.sub-contributors li{ padding-left: 58px; } @@ -1174,3 +1176,12 @@ height: 32px; line-height: 32px; } +li.subproject { + &.FAILED, &.MAXXED { + background-color: $error50 !important; + border-left: 4px solid $error700 !important; + .d-flex .col b, h5, small, p { + color: $error700 !important; + } + } +} \ No newline at end of file diff --git a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx index 29b7cc20b9..62cbf3a450 100644 --- a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx +++ b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.jsx @@ -3,7 +3,7 @@ import React, { useState } from 'react' import { connect } from 'react-redux' import { Icon, Collapse, notification, Typography, Row, Col } from 'antd' import { useTranslation } from 'react-i18next' -import { cloneDeep, isEmpty, orderBy } from 'lodash' +import { cloneDeep, isEmpty } from 'lodash' import classNames from 'classnames' import SimpleMarkdown from 'simple-markdown' diff --git a/akvo/rsr/spa/app/utils/common.scss b/akvo/rsr/spa/app/utils/common.scss index a5db8760ae..1add288ce8 100644 --- a/akvo/rsr/spa/app/utils/common.scss +++ b/akvo/rsr/spa/app/utils/common.scss @@ -1,5 +1,5 @@ $primaryColor: #43998f; -$colors: #560764, #905E96, #D58BDD, #FF99D7, #FFB99A, #CFB495, #8BBABB, #6C7B95, #32AFA9, #A4D4AE, #698474, #889E81, #839b60, #F8B195, #F67280, #C06C84, #6C5B7B, #6C7261, #E6A157, #EB8242, #C9753D, #e9a5cf, #ca80b1, #F0CF85, #381460, #933B5B, #FF6F5E, #40BFC1, #9ACEFF, #12CAD6, #0FABBC, #151965, #32407B, #515585, #46B5D1, #E9EA77, #FFD369, #a99260, #c38a6d, #d8d13e; +$colors: #560764, #905E96, #EB6440, #497174, #FFB99A, #CFB495, #8BBABB, #6C7B95, #32AFA9, #A4D4AE, #698474, #889E81, #839b60, #F8B195, #F67280, #C06C84, #6C5B7B, #6C7261, #E6A157, #EB8242, #C9753D, #e9a5cf, #ca80b1, #F0CF85, #381460, #933B5B, #FF6F5E, #40BFC1, #9ACEFF, #12CAD6, #0FABBC, #151965, #32407B, #515585, #46B5D1, #E9EA77, #FFD369, #a99260, #c38a6d, #d8d13e; $g1colors: #8E936D, #598381, #177E89, #08605F, #75894a, #5C80BC, #4D5061, #30323D, #E8C547, #A2AD59; $g2colors: #0c7b93, #00a8cc, #ddba01, #b0a160, #434e52, #5b8c85, #8d6e63, #ea9085, #92cad3, #6d9499; $g3colors: #2c498b, #35619b, #3e78ab, #4891bb, #52aacb, #6abdd0, #8ecccc, #b4dbcb, #8ca26f; From a1ddaecd6685c4268ac9b78dac63c2dfac90e14b Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Fri, 11 Nov 2022 12:38:48 +0700 Subject: [PATCH 39/59] [#5145] Move aggregation jobs history modal --- .../app/modules/program/AggregatedActual.jsx | 153 +++--------------- .../spa/app/modules/program/Aggregation.jsx | 2 +- .../app/modules/program/AggregationModal.jsx | 138 ++++++++++++++++ .../spa/app/modules/program/ProgramPeriod.jsx | 46 +++--- .../app/modules/program/ProjectSummary.jsx | 3 +- .../rsr/spa/app/modules/program/indicator.jsx | 27 +++- akvo/rsr/spa/app/modules/program/styles.scss | 10 ++ akvo/rsr/spa/app/utils/common.scss | 1 + 8 files changed, 212 insertions(+), 168 deletions(-) create mode 100644 akvo/rsr/spa/app/modules/program/AggregationModal.jsx diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx index 43bbfdc688..a92515c561 100644 --- a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -1,81 +1,28 @@ import React, { useState, useEffect } from 'react' -import { - Button, - Row, - Col, - List, - Modal, - Spin, - Tag, - Tooltip, - Typography, -} from 'antd' +import { Spin } from 'antd' import { connect } from 'react-redux' -import chunk from 'lodash/chunk' -import moment from 'moment' import Icon from '../../components/Icon' import Aggregation from './Aggregation' -import { aggregatedIcons, jobStatusColor, toolTips } from './config' +import { aggregatedIcons } from './config' import { getAllJobByRootPeriod } from './services' import * as actions from './store/actions' -import { printIndicatorPeriod } from '../../utils/dates' - -const { Text, Title } = Typography const AggregatedActual = ({ value, status, amount, total, - periodStart, - periodEnd, periodId, - programmeRdr, - setRootPeriodJobStatus, jobs, + callback, + setRootPeriodJobStatus, }) => { - const [popUp, setPopUp] = useState(false) - const [page, setPage] = useState(0) - const [history, setHistory] = useState([]) const [items, setItems] = useState([]) const [preload, setPreload] = useState({ fetched: (jobs === undefined), created: (jobs === undefined), }) - const pages = chunk(jobs || [], 12) - const _periods = programmeRdr - ?.flatMap((r) => r?.indicators) - ?.flatMap((i) => i?.periods?.map((p) => ({ ...p, indicator: { id: i?.id, title: i?.title } }))) - - const getProjectByPeriodID = (ID) => { - const _contrib = _periods?.flatMap((p) => p?.contributors) - const _contributors = [ - ..._contrib, - ..._contrib?.flatMap((cb) => cb?.contributors) - ] - return _contributors?.find((cb) => cb?.periodId === ID) - } - - const getIndicatorByPeriodID = (ID) => _periods?.find((p) => p?.periodId === ID) - - const handleOnLoadMore = (_page) => { - setPage(_page) - setHistory([...history, ...pages[_page]]) - } - - const LoadMoreButton = () => { - if (jobs && (history.length < jobs.length)) { - return ( -

      - -
      - ) - } - return null - } useEffect(() => { if (items.length === 0 && preload.fetched && preload.created) { @@ -107,85 +54,25 @@ const AggregatedActual = ({ }) setRootPeriodJobStatus(periodId, items) } - if (jobs && history.length === 0) { - setHistory(pages[page]) - } - }, [items, preload, history, jobs]) + }, [items, preload]) const iconType = aggregatedIcons[status] || null - const dataIndicator = getIndicatorByPeriodID(periodId) return ( - <> - - - {iconType && ( - setPopUp(!popUp)}> - - - )} - {(preload.fetched || preload.created) && } spinning />} - - - - {value} - - - - setPopUp(!popUp)} - cancelButtonProps={{ style: { display: 'none' } }} - title="Cron Job History" - width={650} - > -
      - Indicator - - {dataIndicator?.indicator?.title || ''} - - Period - - {printIndicatorPeriod(periodStart, periodEnd)} - -
      - - )} - header={( - Details - )} - renderItem={item => { - const _project = getProjectByPeriodID(item?.period) - return ( - - - - - )} - title={_project?.projectTitle} - description={( - - - {item?.status} - - - {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} - - - )} - /> - - ) - }} - /> -       - + e.stopPropagation()}> + + {iconType && ( + + + + )} + {(preload.fetched || preload.created) && } spinning />} + + + + {value} + + + ) } diff --git a/akvo/rsr/spa/app/modules/program/Aggregation.jsx b/akvo/rsr/spa/app/modules/program/Aggregation.jsx index b76a6cd5c1..0fb8bc6874 100644 --- a/akvo/rsr/spa/app/modules/program/Aggregation.jsx +++ b/akvo/rsr/spa/app/modules/program/Aggregation.jsx @@ -15,7 +15,7 @@ const ContentPopOver = ({ status, callback, amount = 2, total = 85 }) => { description = description?.replace(':value:', `${amount} ${trans('contributor_s', { count: amount })}`) description = description?.replace(':total:', `${total} ${trans('contributor_s', { count: total })}`) const mdParse = SimpleMarkdown.defaultBlockParse - const mdOutput = SimpleMarkdown.defaultOutput + const mdOutput = SimpleMarkdown.defaultReactOutput return ( <>

      {content?.title}

      diff --git a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx new file mode 100644 index 0000000000..73c5568369 --- /dev/null +++ b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx @@ -0,0 +1,138 @@ +import React, { useState, useEffect } from 'react' +import { + Button, + Row, + Col, + List, + Modal, + Tag, + Tooltip, + Typography, +} from 'antd' +import { connect } from 'react-redux' +import chunk from 'lodash/chunk' +import moment from 'moment' + +import Icon from '../../components/Icon' +import { + aggregatedIcons, + jobStatusColor, + toolTips +} from './config' +import { printIndicatorPeriod } from '../../utils/dates' + +const { Text, Title } = Typography + +const AggregationModal = ({ + popUp, + handleOnOk, + programmeRdr, + periodStart, + periodEnd, + periodId, + jobs, +}) => { + const [history, setHistory] = useState([]) + const [page, setPage] = useState(0) + const pages = chunk(jobs || [], 12) + const _periods = programmeRdr + ?.flatMap((r) => r?.indicators) + ?.flatMap((i) => i?.periods?.map((p) => ({ ...p, indicator: { id: i?.id, title: i?.title } }))) + + const getProjectByPeriodID = (ID) => { + const _contrib = _periods?.flatMap((p) => p?.contributors) + const _contributors = [ + ..._contrib, + ..._contrib?.flatMap((cb) => cb?.contributors) + ] + return _contributors?.find((cb) => cb?.periodId === ID) + } + + const getIndicatorByPeriodID = (ID) => _periods?.find((p) => p?.periodId === ID) + + const handleOnLoadMore = (_page) => { + setPage(_page) + setHistory([...history, ...pages[_page]]) + } + + const LoadMoreButton = () => { + if (jobs && (history.length < jobs.length)) { + return ( +
      + +
      + ) + } + return null + } + + const dataIndicator = getIndicatorByPeriodID(periodId) + + useEffect(() => { + if (jobs && history.length === 0) { + setHistory(pages[page]) + } + }, [jobs, history]) + return ( + +
      + Indicator + + {dataIndicator?.indicator?.title || ''} + + Period + + {printIndicatorPeriod(periodStart, periodEnd)} + +
      + + )} + header={( + Details + )} + renderItem={item => { + const _project = getProjectByPeriodID(item?.period) + return ( + + + + + )} + title={_project?.projectTitle} + description={( + + + {item?.status} + + + {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} + + + )} + /> + + ) + }} + /> +       + ) +} + +export default connect( + ({ programmeRdr }) => ({ programmeRdr }), null +)(AggregationModal) diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index e55712bd47..5c6f839860 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -1,6 +1,6 @@ /* global document */ -import React from 'react' -import { Collapse, Row, Col, Select } from 'antd' +import React, { useState } from 'react' +import { Collapse, Select } from 'antd' import moment from 'moment' import classNames from 'classnames' import { useTranslation } from 'react-i18next' @@ -69,27 +69,16 @@ const PeriodHeader = ({ disaggregationContributions, disaggregationTargets, periodId, + callback, jobs, - activePeriods, - setActivePeriods, - periodIndex, }) => { const { t } = useTranslation() - - const handleOnClick = () => { - if (activePeriods?.includes(periodIndex)) { - setActivePeriods(activePeriods?.filter((a) => a !== periodIndex)) - } else { - setActivePeriods([...activePeriods, periodIndex]) - } - } - const groupedStatus = groupBy(jobs || [], 'status') const allStatus = uniq(Object.keys(groupedStatus)) const job = getSummaryStatus(allStatus) return ( -
      event.stopPropagation()}> -
      + <> +
      {printIndicatorPeriod(periodStart, periodEnd)}
      • {filteredContributors.length} {t('contributor_s', { count: filteredContributors.length })}
        • @@ -121,6 +110,7 @@ const PeriodHeader = ({ value={actualValue} amount={groupedStatus[job?.status]?.length || 0} total={filteredContributors?.length} + callback={callback} /> {targetsAt && targetsAt === 'period' && targetValue > 0 && ( @@ -152,7 +142,7 @@ const PeriodHeader = ({
      ) } -
      + ) } @@ -176,11 +166,12 @@ const ProgramPeriod = ({ aggFilteredTotalTarget, aggFilteredTotal, openedItem, - activePeriods, - setActivePeriods, + activePeriod, + setActivePeriod, handleAccordionChange, ...props }) => { + const [popUp, setPopUp] = useState(false) const mouseEnterBar = (index, value, ev) => { if (pinned === index || !listRef.current) return listRef.current.children[0].children[index].classList.add('active') @@ -207,6 +198,13 @@ const ProgramPeriod = ({ } } + const clickOnViewAll = () => { + setActivePeriod({ + period, + popUp: !activePeriod?.popUp + }) + } + const hasDisaggregations = !( period?.disaggregationTargets?.filter(it => it.value).length <= 1 && period?.disaggregationContributions?.filter(it => it.value).length <= 1 @@ -236,10 +234,8 @@ const ProgramPeriod = ({ clickBar, mouseEnterBar, mouseLeaveBar, - activePeriods, - setActivePeriods, - periodIndex, }} + callback={clickOnViewAll} /> )} > @@ -327,12 +323,12 @@ const ProgramPeriod = ({ {subproject.updates.length > 0 &&
      {subproject.updates.length} approved updates
      -
        +
          {subproject.updates.map(update => ( -
        • +
        • {moment(update.createdAt).format('DD MMM YYYY')} {update.user.name} - {update.value && {String(update.value).replace(/\B(?=(\d{3})+(?!\d))/g, ',')}} + {update.value && {setNumberFormat(update.value)}} {update.scoreIndex != null && Score {update.scoreIndex + 1}}
          • ))} diff --git a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx index f0168ce17b..8a5f24c938 100644 --- a/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx +++ b/akvo/rsr/spa/app/modules/program/ProjectSummary.jsx @@ -3,6 +3,7 @@ import moment from 'moment' import { Tooltip } from 'antd' import Icon from '../../components/Icon' import ActualValue from './ActualValue' +import { setNumberFormat } from '../../utils/misc' const getAggregatedUpdatesLength = (updates, contributors) => { let total = 0 @@ -45,7 +46,7 @@ const ProjectSummary = ({
          {updates.length} approved updates
            - {updates.map(update =>
          • {moment(update.createdAt).format('DD MMM YYYY')}{update.user.name}{String(update.value).replace(/\B(?=(\d{3})+(?!\d))/g, ',')}
            • )} + {updates.map(update =>
          • {moment(update.createdAt).format('DD MMM YYYY')}{update.user.name}{setNumberFormat(update.value)}
            • )}
          } diff --git a/akvo/rsr/spa/app/modules/program/indicator.jsx b/akvo/rsr/spa/app/modules/program/indicator.jsx index 62e3b00f64..bab5bb47e2 100644 --- a/akvo/rsr/spa/app/modules/program/indicator.jsx +++ b/akvo/rsr/spa/app/modules/program/indicator.jsx @@ -6,6 +6,7 @@ import TargetCharts from '../../utils/target-charts' import ExpandIcon from './ExpandIcon' import ProgramPeriod from './ProgramPeriod' import { sizes } from './config' +import AggregationModal from './AggregationModal' const Indicator = ({ periods, @@ -18,7 +19,10 @@ const Indicator = ({ const [pinned, setPinned] = useState(-1) const [openedItem, setOpenedItem] = useState(null) const [countriesFilter, setCountriesFilter] = useState([]) - const [activePeriods, setActivePeriods] = useState([]) + const [activePeriod, setActivePeriod] = useState({ + popUp: false, + period: null, + }) const listRef = useRef(null) const pinnedRef = useRef(-1) const tooltipRef = useRef(null) @@ -87,11 +91,7 @@ const Indicator = ({ )} - } - > + }> {periods.map((period, index) => { const filteredContributors = period.contributors.filter(filterProjects) const filteredCountries = countryFilter.length > 0 ? countryFilter : period.countries @@ -121,8 +121,8 @@ const Indicator = ({ countriesFilter, openedItem, periodIndex: index, - activePeriods, - setActivePeriods, + activePeriod, + setActivePeriod, handleCountryFilter: setCountriesFilter, handleAccordionChange }} @@ -130,6 +130,17 @@ const Indicator = ({ ) })} + {activePeriod?.period && ( + setActivePeriod({ + ...activePeriod, + popUp: !activePeriod?.popUp + })} + /> + ) + }
      ) } diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index 92ba0caf85..b684c9c2c4 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -1184,4 +1184,14 @@ li.subproject { color: $error700 !important; } } +} +.updates-popup ul.FAILED li:first-child, +.updates-popup ul.MAXXED li:first-child { + b, span { + color: $error700 !important; + } +} +.updates-popup ul.SCHEDULED li:first-child, +.updates-popup ul.RUNNING li:first-child { + background-color: $warning100; } \ No newline at end of file diff --git a/akvo/rsr/spa/app/utils/common.scss b/akvo/rsr/spa/app/utils/common.scss index 1add288ce8..327fe35e8d 100644 --- a/akvo/rsr/spa/app/utils/common.scss +++ b/akvo/rsr/spa/app/utils/common.scss @@ -8,6 +8,7 @@ $submitted: rgb(24, 144, 255); $error50: #FEF3F2; $error700: #B42318; $primary700: #218EFC; +$warning100: #FEF0C7; @mixin disaggregation-bar-colors{ &:nth-child(1) .disaggregations-bar .dsg-item{ From e787215889443a2184d951f5f6d75728c084c133 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Fri, 11 Nov 2022 15:51:27 +0700 Subject: [PATCH 40/59] [#5145] Endpoint to restart indicator aggregation job --- .../views/indicator_period_aggregation_job.py | 21 +++++++ .../jobs/test_indicator_period_aggregation.py | 57 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/akvo/rest/views/indicator_period_aggregation_job.py b/akvo/rest/views/indicator_period_aggregation_job.py index d4dd690261..3f3be6ecdd 100644 --- a/akvo/rest/views/indicator_period_aggregation_job.py +++ b/akvo/rest/views/indicator_period_aggregation_job.py @@ -4,13 +4,17 @@ # See more details in the license.txt file located at the root folder of the Akvo RSR module. # For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >. from rest_framework import filters +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response from akvo.rsr.models import IndicatorPeriodAggregationJob from ..filters import RSRGenericFilterBackend from ..serializers import IndicatorPeriodAggregationJobSerializer from ..viewsets import ReadOnlyPublicProjectViewSet, SafeMethodsPermissions +from ...rsr.usecases.jobs.aggregation import schedule_aggregation_job class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet): @@ -24,3 +28,20 @@ class IndicatorPeriodAggregationJobViewSet(ReadOnlyPublicProjectViewSet): # These are login only resources that shouldn't be interesting to the public permission_classes = (SafeMethodsPermissions, IsAuthenticated) + + @action(detail=True, methods=['post']) + def reschedule(self, request, **kwargs): + """ + Puts a new job in the queue for the indicator period + + The old job is left in order to have a history + """ + job: IndicatorPeriodAggregationJob = self.get_object() + + if job.status != IndicatorPeriodAggregationJob.Status.MAXXED: + raise ValidationError("Maximum number of attempts not reached") + + new_job = schedule_aggregation_job(job.period) + serializer = self.get_serializer(new_job) + + return Response(serializer.data) diff --git a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py index e026fd359d..d03de0b858 100644 --- a/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py +++ b/akvo/rsr/tests/rest/jobs/test_indicator_period_aggregation.py @@ -8,7 +8,7 @@ """ from unittest.mock import ANY -from rest_framework.status import HTTP_200_OK, HTTP_403_FORBIDDEN +from rest_framework.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_403_FORBIDDEN from akvo.rsr.models import IndicatorPeriodAggregationJob from akvo.rsr.permissions import GROUP_NAME_ME_MANAGERS @@ -127,3 +127,58 @@ def test_get_by_id(self): data = response.json() self.assertEqual(data["id"], self.private_job.id) + + def test_reschedule(self): + """Ensure rescheduling creates a new job and leaves the old one intact""" + + self.c.login(username=self.user.username, password="password") + self.private_job.mark_maxxed() + + response = self.c.post( + f"/rest/v1/jobs/indicator_period_aggregation/{self.private_job.id}/reschedule/?format=json" + ) + + self.assertEqual(response.status_code, HTTP_200_OK) + + data = response.json() + self.assertNotEqual(data["id"], self.private_job.id) + + self.assertEqual( + IndicatorPeriodAggregationJob.objects.filter(period=self.private_period).count(), + 2 + ) + + def test_reschedule_unmaxxed_job(self): + """Attempting to reschedule a job in the wrong status shouldn't be allowed""" + + self.c.login(username=self.user.username, password="password") + + response = self.c.post( + f"/rest/v1/jobs/indicator_period_aggregation/{self.private_job.id}/reschedule/?format=json" + ) + + self.assertEqual((response.status_code, response.content), (HTTP_400_BAD_REQUEST, ANY)) + + def test_reschedule_private_from_other_user(self): + """Attempting a reschedule of a private job from a user of another org should fail""" + + self.c.login(username=self.other_private_user.username, password="password") + self.private_job.mark_maxxed() + + response = self.c.post( + f"/rest/v1/jobs/indicator_period_aggregation/{self.private_job.id}/reschedule/?format=json" + ) + + self.assertEqual(response.status_code, HTTP_403_FORBIDDEN) + + def test_reschedule_public_from_other_user(self): + """Attempting a reschedule of a public job from a user of another org should fail""" + + self.c.login(username=self.other_private_user.username, password="password") + self.job.mark_maxxed() + + response = self.c.post( + f"/rest/v1/jobs/indicator_period_aggregation/{self.job.id}/reschedule/?format=json" + ) + + self.assertEqual(response.status_code, HTTP_403_FORBIDDEN) From 04c67d44f9aee6048345be32b9c0f2c6fbe45979 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Fri, 11 Nov 2022 19:49:49 +0700 Subject: [PATCH 41/59] [#5145] Make restart endpoint and handle maxxed failed job --- .../tests/usecases/jobs/test_aggregation.py | 21 +++++++++++++++++++ akvo/rsr/usecases/jobs/aggregation.py | 9 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index bcf89831f1..c1f592b066 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -6,6 +6,7 @@ from akvo.rsr.models import Indicator, IndicatorPeriod, Result, User from akvo.rsr.models.aggregation_job import IndicatorPeriodAggregationJob +from akvo.rsr.models.cron_job import CronJobMixin from akvo.rsr.permissions import GROUP_NAME_ME_MANAGERS from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.usecases.jobs import aggregation as usecases @@ -211,3 +212,23 @@ def test_success_after_failure(self): msg = mail.outbox[1] self.assertEqual(msg.to, [self.user.email]) self.assertEqual(msg.subject, "Previously failed indicator aggregation job has succeeded") + + +class HandleFailedJobTestCase(AggregationJobBaseTests): + def setUp(self): + super().setUp() + for _ in range(usecases.MAX_ATTEMPTS - 1): + self.job.mark_failed() + + def test_mark_scheduled(self): + usecases.handle_failed_jobs() + self.job.refresh_from_db() + self.assertEqual(4, self.job.attempts) + self.assertEqual(CronJobMixin.Status.SCHEDULED, self.job.status) + + def test_mark_maxxed(self): + self.job.mark_failed() + usecases.handle_failed_jobs() + self.job.refresh_from_db() + self.assertEqual(5, self.job.attempts) + self.assertEqual(CronJobMixin.Status.MAXXED, self.job.status) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index a2d32d7034..938f99077d 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -18,6 +18,8 @@ logger = logging.getLogger(__name__) +MAX_ATTEMPTS = 5 + def get_scheduled_jobs() -> QuerySet[IndicatorPeriodAggregationJob]: return base_get_jobs().filter(status=CronJobMixin.Status.SCHEDULED) @@ -94,10 +96,13 @@ def run_aggregation(period: IndicatorPeriod): @atomic def handle_failed_jobs(): - """Identify failed jobs and reschedule them""" + """Identify failed jobs and reschedule them up to max attempts""" fail_dead_jobs() for failed_job in get_failed_jobs(): - failed_job.mark_scheduled() + if failed_job.attempts < MAX_ATTEMPTS: + failed_job.mark_scheduled() + else: + failed_job.mark_maxxed() @atomic From e62eb1b509574c4707883dc22d14cfdc2396af3a Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Mon, 14 Nov 2022 10:11:43 +0700 Subject: [PATCH 42/59] [#5145] implementing restart aggregation jobs endpoint Add new actionTypes `UPDATE_JOB_STATUS` to update job status from MAXXED to SCHEDULED after calling the endpoint /jobs/indicator_period_aggregation/${jobID}/reschedule/?format=json --- .../spa/app/modules/program/ActualValue.jsx | 32 +++++++++++++++---- .../app/modules/program/AggregationModal.jsx | 2 +- .../spa/app/modules/program/ProgramPeriod.jsx | 4 ++- .../app/modules/program/store/action-types.js | 1 + .../spa/app/modules/program/store/actions.js | 4 +++ .../spa/app/modules/program/store/reducer.js | 30 +++++++++++++++++ 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/ActualValue.jsx b/akvo/rsr/spa/app/modules/program/ActualValue.jsx index cfa5b6c2de..e88c9f56ab 100644 --- a/akvo/rsr/spa/app/modules/program/ActualValue.jsx +++ b/akvo/rsr/spa/app/modules/program/ActualValue.jsx @@ -1,5 +1,7 @@ -import React from 'react' -import { Button } from 'antd' +import React, { useState } from 'react' +import { Button, message } from 'antd' +import { connect } from 'react-redux' + import Icon from '../../components/Icon' import Aggregation from './Aggregation' import { @@ -8,17 +10,31 @@ import { jobStatus, toolTips } from './config' +import api from '../../utils/api' +import * as actions from './store/actions' const ActualValue = ({ + updateJobStatus, actualValue, job = {}, }) => { + const [loading, setLoading] = useState(false) const _status = (!job?.id && job?.status === jobStatus.maxxed) ? jobStatus.failed : job?.status const title = toolTips[_status] || null const iconType = actualValueIcons[_status] || null - const handleOnRestartJob = () => { - console.log('call to API') + const handleOnRestartJob = (jobID) => { + setLoading(true) + api + .post(`/jobs/indicator_period_aggregation/${jobID}/reschedule/?format=json`) + .then(({ data }) => { + setLoading(false) + updateJobStatus(jobID, data) + }) + .catch((err) => { + setLoading(false) + if (err) message.error('Failed to restart the job') + }) } return ( @@ -29,8 +45,8 @@ const ActualValue = ({ { (callToAction.includes(job?.status) && job?.id) ? ( - ) : ( @@ -49,4 +65,6 @@ const ActualValue = ({ ) } -export default ActualValue +export default connect( + null, actions +)(ActualValue) diff --git a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx index 73c5568369..d84f2a8dc4 100644 --- a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx @@ -119,7 +119,7 @@ const AggregationModal = ({ {item?.status} - + {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} diff --git a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx index 5c6f839860..725c480f68 100644 --- a/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx +++ b/akvo/rsr/spa/app/modules/program/ProgramPeriod.jsx @@ -73,7 +73,9 @@ const PeriodHeader = ({ jobs, }) => { const { t } = useTranslation() - const groupedStatus = groupBy(jobs || [], 'status') + const jobsPerPeriod = jobs ? groupBy(jobs, 'period') : [] + const latestJobs = Object?.values(jobsPerPeriod)?.map((values) => values?.shift()) + const groupedStatus = groupBy(latestJobs, 'status') const allStatus = uniq(Object.keys(groupedStatus)) const job = getSummaryStatus(allStatus) return ( diff --git a/akvo/rsr/spa/app/modules/program/store/action-types.js b/akvo/rsr/spa/app/modules/program/store/action-types.js index 3d22b93d3f..1bc2e1bb72 100644 --- a/akvo/rsr/spa/app/modules/program/store/action-types.js +++ b/akvo/rsr/spa/app/modules/program/store/action-types.js @@ -2,4 +2,5 @@ export default { APPEND_RESULTS: 'PG_APPEND_RESULTS', UPDATE_RESULT: 'PG_UPDATE_RESULT', SET_JOB_STATUS: 'PG_SET_JOB_STATUS', + UPDATE_JOB_STATUS: 'PG_UPDATE_JOB_STATUS', } diff --git a/akvo/rsr/spa/app/modules/program/store/actions.js b/akvo/rsr/spa/app/modules/program/store/actions.js index ac4dc53d60..070593947d 100644 --- a/akvo/rsr/spa/app/modules/program/store/actions.js +++ b/akvo/rsr/spa/app/modules/program/store/actions.js @@ -11,3 +11,7 @@ export const updateProgrammePerResult = (resultIndex, data) => (dispatch) => { export const setRootPeriodJobStatus = (rootPeriod, results) => (dispatch) => { dispatch({ type: actionTypes.SET_JOB_STATUS, payload: { rootPeriod, results } }) } + +export const updateJobStatus = (jobID, data) => (dispatch) => { + dispatch({ type: actionTypes.UPDATE_JOB_STATUS, payload: { jobID, data } }) +} diff --git a/akvo/rsr/spa/app/modules/program/store/reducer.js b/akvo/rsr/spa/app/modules/program/store/reducer.js index e06f3cbfcf..2e392cc688 100644 --- a/akvo/rsr/spa/app/modules/program/store/reducer.js +++ b/akvo/rsr/spa/app/modules/program/store/reducer.js @@ -56,6 +56,36 @@ export default (state = [], action) => { }) })) })) + case actionTypes.UPDATE_JOB_STATUS: + const { jobID, data: theJob } = action.payload + return state?.map((s) => ({ + ...s, + indicators: s?.indicators?.map((i) => ({ + ...i, + periods: i?.periods?.map((p) => ({ + ...p, + jobs: p?.jobs ? [theJob, ...p.jobs] : undefined, + contributors: p?.contributors?.map((cb) => { + const _subContributors = cb?.contributors?.map((subC) => { + if (subC?.job?.id === jobID) { + return ({ + ...subC, + job: theJob + }) + } + return subC + }) + const allStatus = uniq(_subContributors?.map((subC) => subC?.job?.status))?.filter((status) => status) + const job = (cb?.job?.id === jobID) ? theJob : getSummaryStatus(allStatus) + return ({ + ...cb, + job, + contributors: _subContributors + }) + }) + })) + })) + })) default: return state } From ff69da32b62cf94819607988221a2ab7c74133bb Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Mon, 14 Nov 2022 19:39:34 +0700 Subject: [PATCH 43/59] [#5145] Fix wording in tooltips, icon, modal and list --- .../spa/app/modules/program/ActualValue.jsx | 30 ++++++++++--------- .../app/modules/program/AggregationModal.jsx | 14 +++++---- akvo/rsr/spa/app/modules/program/config.js | 20 +++++-------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/ActualValue.jsx b/akvo/rsr/spa/app/modules/program/ActualValue.jsx index e88c9f56ab..7d6472e44b 100644 --- a/akvo/rsr/spa/app/modules/program/ActualValue.jsx +++ b/akvo/rsr/spa/app/modules/program/ActualValue.jsx @@ -5,9 +5,9 @@ import { connect } from 'react-redux' import Icon from '../../components/Icon' import Aggregation from './Aggregation' import { - actualValueIcons, + aggregatedIcons, callToAction, - jobStatus, + MAX_ATTEMPTS, toolTips } from './config' import api from '../../utils/api' @@ -19,9 +19,9 @@ const ActualValue = ({ job = {}, }) => { const [loading, setLoading] = useState(false) - const _status = (!job?.id && job?.status === jobStatus.maxxed) ? jobStatus.failed : job?.status + const _status = job?.status const title = toolTips[_status] || null - const iconType = actualValueIcons[_status] || null + const iconType = aggregatedIcons[_status] || null const handleOnRestartJob = (jobID) => { setLoading(true) @@ -41,19 +41,21 @@ const ActualValue = ({ {_status && ( - - { - (callToAction.includes(job?.status) && job?.id) - ? ( + { + (callToAction.includes(job?.status) && job?.id && job?.attempts < MAX_ATTEMPTS) + ? ( + - ) - : ( + + ) + : ( + - ) - } - + + ) + } )} diff --git a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx index d84f2a8dc4..5d49804f9e 100644 --- a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx @@ -81,7 +81,7 @@ const AggregationModal = ({ visible={popUp} onOk={handleOnOk} cancelButtonProps={{ style: { display: 'none' } }} - title="Cron Job History" + title="Aggregation Job History" width={650} >
      @@ -113,15 +113,17 @@ const AggregationModal = ({ )} - title={_project?.projectTitle} + title={( + <> + {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} + {` - ${_project?.projectTitle}`} + + )} description={( - + {item?.status} - - {moment(item?.updatedAt).format('DD MMM YYYY H:mm:ss')} - )} /> diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index cf25c0e4e5..dd3eea2ebc 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -38,14 +38,6 @@ export const aggregatedIcons = { MAXXED: 'rsr.circle.alert', } -export const actualValueIcons = { - SCHEDULED: 'rsr.clock', - RUNNING: 'rsr.loader', - FINISHED: 'rsr.circle.check', - FAILED: 'rsr.circle.alert', - MAXXED: 'rsr.repeat', -} - export const jobStatusColor = { SCHEDULED: '#667085', RUNNING: 'gold', @@ -56,19 +48,19 @@ export const jobStatusColor = { export const popOver = { SCHEDULED: { - title: 'Cron Job Scheduled', + title: 'Job Scheduled', description: 'Aggregation is scheduled', }, RUNNING: { - title: 'Cron Job Running', + title: 'Job Running', description: 'Aggregation in progress', }, FINISHED: { - title: 'Cron Job Finished', + title: 'Job Finished', description: 'Aggregated value is valid', }, FAILED: { - title: 'Cron Job Failed', + title: 'Job Failed', description: '**:value:** out of **:total:** failed to update', action: 'view all' }, @@ -84,9 +76,11 @@ export const toolTips = { RUNNING: popOver.RUNNING.title, FINISHED: popOver.FINISHED.title, FAILED: popOver.FAILED.title, - MAXXED: 'Restart the job', + MAXXED: 'Too many failed attempts', } export const callToAction = [ jobStatus.maxxed, ] + +export const MAX_ATTEMPTS = 5 From c9d91982b02e144debca7ca204e32bde55a41e50 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 14 Nov 2022 14:23:09 +0100 Subject: [PATCH 44/59] fix: aggregation job emails #5158: [#5145] Maxxed and restart aggregation jobs --- akvo/rsr/usecases/jobs/aggregation.py | 10 ++++++---- akvo/templates/indicator_aggregation/fail_message.html | 8 ++++---- .../indicator_aggregation/success_message.html | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/akvo/rsr/usecases/jobs/aggregation.py b/akvo/rsr/usecases/jobs/aggregation.py index 938f99077d..3ddd9366be 100644 --- a/akvo/rsr/usecases/jobs/aggregation.py +++ b/akvo/rsr/usecases/jobs/aggregation.py @@ -134,19 +134,21 @@ def fail_dead_jobs() -> List[IndicatorPeriodAggregationJob]: def email_job_owners( - failed_job: IndicatorPeriodAggregationJob, + job: IndicatorPeriodAggregationJob, subject_template: str, message_template: str, reason: str = None ): - recipients = get_job_recipients(failed_job) + recipients = get_job_recipients(job) rsr_send_mail_to_users( [recipient.user for recipient in recipients], subject=subject_template, message=message_template, msg_context={ - "indicator": failed_job.period.indicator, - "root_project": failed_job.root_project, + "indicator": job.period.indicator, + "root_project": job.root_project, "reason": reason, + "job": job, + "max_attempts": MAX_ATTEMPTS, } ) diff --git a/akvo/templates/indicator_aggregation/fail_message.html b/akvo/templates/indicator_aggregation/fail_message.html index 74bd259ef6..0d39a154a1 100644 --- a/akvo/templates/indicator_aggregation/fail_message.html +++ b/akvo/templates/indicator_aggregation/fail_message.html @@ -14,17 +14,17 @@ {% blocktrans %} -

      The indicator "{{ indicator.title }}" in the hierarchy of the program/project +

      The indicator: "{{ indicator.title }}" in the hierarchy of the program/project {{ root_project.title }} - had triggered an aggregation job. + had triggered a job that had failed.

      Reason: {{ reason }}

      -

      The job failed and couldn't complete aggregation.

      +

      The job failed and couldn't complete aggregation {{ job.attempts }} of {{ max_attempts }} times.

      It shall be rerun shortly.

      -

      You shall receive an email once the indicator has been successfully updated

      +

      You shall receive an email once the indicator has been successfully updated.

      {% endblocktrans %} diff --git a/akvo/templates/indicator_aggregation/success_message.html b/akvo/templates/indicator_aggregation/success_message.html index 394d17cbeb..4f37150dc3 100644 --- a/akvo/templates/indicator_aggregation/success_message.html +++ b/akvo/templates/indicator_aggregation/success_message.html @@ -14,9 +14,9 @@ {% blocktrans %} -

      The indicator: "{{ indicator.title }}" in the hierarchy of the program/project +

      The indicator "{{ indicator.title }}" in the hierarchy of the program/project {{ root_project.title }} - had triggered a job that had failed. + had triggered an aggregation job.

      After rerunning the job, it has now succeeded and the aggregation is complete.

      From 83134479c7128c4ff625cabad38ebecc925b8347 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Tue, 15 Nov 2022 09:55:45 +0700 Subject: [PATCH 45/59] [#5145] Fix get the latest job and remove logic max_attemps --- .../spa/app/modules/program/ActualValue.jsx | 5 ++--- .../app/modules/program/AggregatedActual.jsx | 20 +++++++++++++------ .../app/modules/program/AggregationModal.jsx | 20 ++++--------------- .../spa/app/modules/program/ProgramPeriod.jsx | 10 ---------- akvo/rsr/spa/app/modules/program/config.js | 2 -- akvo/rsr/spa/app/modules/program/services.js | 18 +++++++++++++++++ .../spa/app/modules/program/store/reducer.js | 10 +++++----- 7 files changed, 43 insertions(+), 42 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/ActualValue.jsx b/akvo/rsr/spa/app/modules/program/ActualValue.jsx index 7d6472e44b..887cf841c0 100644 --- a/akvo/rsr/spa/app/modules/program/ActualValue.jsx +++ b/akvo/rsr/spa/app/modules/program/ActualValue.jsx @@ -7,7 +7,6 @@ import Aggregation from './Aggregation' import { aggregatedIcons, callToAction, - MAX_ATTEMPTS, toolTips } from './config' import api from '../../utils/api' @@ -42,10 +41,10 @@ const ActualValue = ({ {_status && ( { - (callToAction.includes(job?.status) && job?.id && job?.attempts < MAX_ATTEMPTS) + (callToAction.includes(job?.status) && job?.id) ? ( - diff --git a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx index a92515c561..880c369037 100644 --- a/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregatedActual.jsx @@ -1,21 +1,21 @@ import React, { useState, useEffect } from 'react' import { Spin } from 'antd' import { connect } from 'react-redux' +import { groupBy, uniq } from 'lodash' import Icon from '../../components/Icon' import Aggregation from './Aggregation' import { aggregatedIcons } from './config' -import { getAllJobByRootPeriod } from './services' +import { getAllJobByRootPeriod, getAllPeriods, getProjectByPeriodID, getSummaryStatus } from './services' import * as actions from './store/actions' const AggregatedActual = ({ value, - status, - amount, total, periodId, jobs, callback, + programmeRdr, setRootPeriodJobStatus, }) => { const [items, setItems] = useState([]) @@ -56,13 +56,21 @@ const AggregatedActual = ({ } }, [items, preload]) - const iconType = aggregatedIcons[status] || null + const _periods = getAllPeriods(programmeRdr) + const _projects = jobs?.length ? jobs?.map((j) => getProjectByPeriodID(_periods, j?.period)) : [] + const _projectGrouped = groupBy(_projects, 'projectId') + const _latestJobs = Object.values(_projectGrouped)?.map((p) => p?.shift())?.map((p) => p?.job) + const groupedStatus = groupBy(_latestJobs, 'status') + const allStatus = uniq(Object.keys(groupedStatus)) + const job = getSummaryStatus(allStatus) + const jobAmount = groupedStatus[job?.status]?.length || 0 + const iconType = aggregatedIcons[job?.status] || null return ( e.stopPropagation()}> {iconType && ( - - + + )} {(preload.fetched || preload.created) && } spinning />} diff --git a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx index 5d49804f9e..b8a20e2e51 100644 --- a/akvo/rsr/spa/app/modules/program/AggregationModal.jsx +++ b/akvo/rsr/spa/app/modules/program/AggregationModal.jsx @@ -20,6 +20,7 @@ import { toolTips } from './config' import { printIndicatorPeriod } from '../../utils/dates' +import { getAllPeriods, getIndicatorByPeriodID, getProjectByPeriodID } from './services' const { Text, Title } = Typography @@ -35,20 +36,7 @@ const AggregationModal = ({ const [history, setHistory] = useState([]) const [page, setPage] = useState(0) const pages = chunk(jobs || [], 12) - const _periods = programmeRdr - ?.flatMap((r) => r?.indicators) - ?.flatMap((i) => i?.periods?.map((p) => ({ ...p, indicator: { id: i?.id, title: i?.title } }))) - - const getProjectByPeriodID = (ID) => { - const _contrib = _periods?.flatMap((p) => p?.contributors) - const _contributors = [ - ..._contrib, - ..._contrib?.flatMap((cb) => cb?.contributors) - ] - return _contributors?.find((cb) => cb?.periodId === ID) - } - - const getIndicatorByPeriodID = (ID) => _periods?.find((p) => p?.periodId === ID) + const _periods = getAllPeriods(programmeRdr) const handleOnLoadMore = (_page) => { setPage(_page) @@ -68,7 +56,7 @@ const AggregationModal = ({ return null } - const dataIndicator = getIndicatorByPeriodID(periodId) + const dataIndicator = getIndicatorByPeriodID(_periods, periodId) useEffect(() => { if (jobs && history.length === 0) { @@ -104,7 +92,7 @@ const AggregationModal = ({ Details )} renderItem={item => { - const _project = getProjectByPeriodID(item?.period) + const _project = getProjectByPeriodID(_periods, item?.period) return ( { const { t } = useTranslation() - const jobsPerPeriod = jobs ? groupBy(jobs, 'period') : [] - const latestJobs = Object?.values(jobsPerPeriod)?.map((values) => values?.shift()) - const groupedStatus = groupBy(latestJobs, 'status') - const allStatus = uniq(Object.keys(groupedStatus)) - const job = getSummaryStatus(allStatus) return ( <>
      @@ -102,7 +94,6 @@ const PeriodHeader = ({
      aggregated actual
      diff --git a/akvo/rsr/spa/app/modules/program/config.js b/akvo/rsr/spa/app/modules/program/config.js index dd3eea2ebc..002f4f5376 100644 --- a/akvo/rsr/spa/app/modules/program/config.js +++ b/akvo/rsr/spa/app/modules/program/config.js @@ -82,5 +82,3 @@ export const toolTips = { export const callToAction = [ jobStatus.maxxed, ] - -export const MAX_ATTEMPTS = 5 diff --git a/akvo/rsr/spa/app/modules/program/services.js b/akvo/rsr/spa/app/modules/program/services.js index 5d545a3ddf..a78785a8c7 100644 --- a/akvo/rsr/spa/app/modules/program/services.js +++ b/akvo/rsr/spa/app/modules/program/services.js @@ -39,3 +39,21 @@ export const getAllResponse = async (responses, callback) => { data = data?.flatMap((d) => d) if (callback) callback(data) } + +export const getAllPeriods = (results) => { + const _periods = results + ?.flatMap((r) => r?.indicators) + ?.flatMap((i) => i?.periods?.map((p) => ({ ...p, indicator: { id: i?.id, title: i?.title } }))) + return _periods +} + +export const getProjectByPeriodID = (_periods, ID) => { + const _contrib = _periods?.flatMap((p) => p?.contributors) + const _contributors = [ + ..._contrib, + ..._contrib?.flatMap((cb) => cb?.contributors) + ] + return _contributors?.find((cb) => cb?.periodId === ID) +} + +export const getIndicatorByPeriodID = (_periods, ID) => _periods?.find((p) => p?.periodId === ID) diff --git a/akvo/rsr/spa/app/modules/program/store/reducer.js b/akvo/rsr/spa/app/modules/program/store/reducer.js index 2e392cc688..2a438ccdea 100644 --- a/akvo/rsr/spa/app/modules/program/store/reducer.js +++ b/akvo/rsr/spa/app/modules/program/store/reducer.js @@ -23,13 +23,14 @@ export default (state = [], action) => { ...i, periods: i?.periods?.map((p) => { if (p?.periodId === rootPeriod) { + const jobs = orderBy(results?.filter((r) => (r?.status)), ['id'], ['desc']) const _contributors = p?.contributors?.map((cb) => { // parent contributor - const parentJobs = results?.filter((rs) => rs?.period === cb?.periodId) + const parentJobs = jobs?.filter((j) => j?.period === cb?.periodId) const _subContributors = cb?.contributors?.map((subCb) => { // child contributor - const jobs = results?.filter((rs) => rs?.period === subCb?.periodId) - const latestJob = jobs?.pop() || {} + const _jobs = jobs?.filter((j) => j?.period === subCb?.periodId) + const latestJob = _jobs?.shift() || {} return ({ ...subCb, job: latestJob @@ -37,7 +38,7 @@ export default (state = [], action) => { }) const allStatus = uniq(_subContributors?.map((subC) => subC?.job?.status))?.filter((status) => status) const job = parentJobs?.length - ? parentJobs.pop() + ? parentJobs.shift() : getSummaryStatus(allStatus) return ({ ...cb, @@ -45,7 +46,6 @@ export default (state = [], action) => { contributors: _subContributors }) }) - const jobs = orderBy(results?.filter((r) => (r?.status)), ['updatedAt'], ['desc']) return ({ ...p, jobs, From 7af476bd9a52caafb615059233e52bd7bd523700 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Tue, 15 Nov 2022 12:30:09 +0700 Subject: [PATCH 46/59] [#5145] Refactor HandleFailedJobTestCase --- akvo/rsr/tests/usecases/jobs/test_aggregation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/akvo/rsr/tests/usecases/jobs/test_aggregation.py b/akvo/rsr/tests/usecases/jobs/test_aggregation.py index c1f592b066..7ac7e132b7 100644 --- a/akvo/rsr/tests/usecases/jobs/test_aggregation.py +++ b/akvo/rsr/tests/usecases/jobs/test_aggregation.py @@ -217,8 +217,9 @@ def test_success_after_failure(self): class HandleFailedJobTestCase(AggregationJobBaseTests): def setUp(self): super().setUp() - for _ in range(usecases.MAX_ATTEMPTS - 1): - self.job.mark_failed() + self.job.status = IndicatorPeriodAggregationJob.Status.FAILED + self.job.attempts = usecases.MAX_ATTEMPTS - 1 + self.job.save() def test_mark_scheduled(self): usecases.handle_failed_jobs() @@ -227,7 +228,8 @@ def test_mark_scheduled(self): self.assertEqual(CronJobMixin.Status.SCHEDULED, self.job.status) def test_mark_maxxed(self): - self.job.mark_failed() + self.job.attempts = usecases.MAX_ATTEMPTS + self.job.save() usecases.handle_failed_jobs() self.job.refresh_from_db() self.assertEqual(5, self.job.attempts) From d7aae37a9ba62d6161fd8f492d151803414732f8 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Tue, 15 Nov 2022 18:10:47 +0700 Subject: [PATCH 47/59] [#5145] Remove text color in sub-contributor(s) list This aims to avoid error indications when the text color is close to red. --- akvo/rsr/spa/app/modules/program/styles.scss | 9 --------- 1 file changed, 9 deletions(-) diff --git a/akvo/rsr/spa/app/modules/program/styles.scss b/akvo/rsr/spa/app/modules/program/styles.scss index b684c9c2c4..44c4a26eba 100644 --- a/akvo/rsr/spa/app/modules/program/styles.scss +++ b/akvo/rsr/spa/app/modules/program/styles.scss @@ -543,15 +543,6 @@ @for $i from 1 through 40 { &:nth-of-type(#{$i}){ border-color: nth($colors, $i); - ul.sub-contributors{ - &>li{ - .value{ - b, small{ - color: darken(nth($colors, $i), 23%); - } - } - } - } } } } From 4a21ba94d607b28b85b24b90c5dcbc2172365e2b Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Wed, 16 Nov 2022 19:23:34 +0700 Subject: [PATCH 48/59] [#5161] feature reporting on negative percentages - Remove positive value validation at the inputNumber (final-field component) in the front-end side. - Add a new constant variable to replace hard-coded checking the `indicator.measure` value. - Fix negative value on percentage indicator disaggregation. --- akvo/rsr/spa/app/components/AddUpdate.jsx | 17 +- akvo/rsr/spa/app/components/LineChart.jsx | 4 +- akvo/rsr/spa/app/components/PrevUpdate.jsx | 5 +- .../app/modules/results-admin/ResultAdmin.jsx | 3 +- .../components/QuantitativeIndicator.jsx | 3 +- .../results-admin/components/ReportedEdit.jsx | 3 - .../results-admin/components/ReportedForm.jsx | 331 +++++++++--------- .../results-overview/ResultOverview.scss | 18 +- .../components/UpdateItems.jsx | 3 +- .../app/modules/results/EnumeratorPage.jsx | 3 +- .../spa/app/modules/results/edit-update.jsx | 7 +- akvo/rsr/spa/app/modules/results/period.jsx | 7 +- akvo/rsr/spa/app/modules/results/update.jsx | 3 +- akvo/rsr/spa/app/utils/constants.js | 5 + akvo/rsr/spa/app/utils/final-field.jsx | 2 +- akvo/rsr/spa/app/utils/misc.js | 13 + 16 files changed, 233 insertions(+), 194 deletions(-) diff --git a/akvo/rsr/spa/app/components/AddUpdate.jsx b/akvo/rsr/spa/app/components/AddUpdate.jsx index c6383240b1..25e3c787b6 100644 --- a/akvo/rsr/spa/app/components/AddUpdate.jsx +++ b/akvo/rsr/spa/app/components/AddUpdate.jsx @@ -20,6 +20,7 @@ import { PrevUpdate } from './PrevUpdate' import ScoringField from './ScoringField' import { StatusUpdate } from './StatusUpdate' import LineChart from './LineChart' +import { measureType } from '../utils/constants' const axiosConfig = { headers: { ...config.headers, 'Content-Type': 'multipart/form-data' }, @@ -64,7 +65,7 @@ export const AddUpdate = ({ setFileSet([]) }, [period]) const draftUpdate = period.updates.find(it => it.status === 'D') - const pendingUpdate = (period.updates[0]?.status === 'P' || (indicator.measure === '2' && period.updates[0]?.status !== 'R')/* trick % measure update to show as "pending update" */) ? period.updates[0] : null + const pendingUpdate = (period.updates[0]?.status === 'P' || (indicator.measure === measureType.PERCENTAGE && period.updates[0]?.status !== 'R')/* trick % measure update to show as "pending update" */) ? period.updates[0] : null const recentUpdate = /* in the last 12 hours AND NOT returned for revision */ period.updates.filter(it => it.status !== 'R').find(it => { const minDiff = (new Date().getTime() - new Date(it.lastModifiedAt).getTime()) / 60000; return minDiff < 720 }) // the above is used for the M&E view bc their value updates skip the "pending" status const submittedUpdate = pendingUpdate || recentUpdate @@ -282,7 +283,7 @@ export const AddUpdate = ({
      {group.name}
      {group.dimensionValues.map(dsg => { - return indicator.measure === '1' ? ( + return indicator.measure === measureType.UNIT ? ( it.typeId === dsg.id && group.id === it.groupId)}].value`} control="input-number" @@ -331,11 +332,11 @@ export const AddUpdate = ({ if (item.denominator) dsgGroups[item.category].denominator += item.denominator }) const categories = Object.keys(dsgGroups) - if (categories.length > 0 && indicator.measure === '1') { + if (categories.length > 0 && indicator.measure === measureType.UNIT) { const value = categories.reduce((acc, key) => dsgGroups[key].value > acc ? dsgGroups[key].value : acc, 0) if (value > 0) form.change('value', value) } - if (categories.length > 0 && indicator.measure === '2') { + if (categories.length > 0 && indicator.measure === measureType.PERCENTAGE) { const [numerator, denominator] = categories.reduce(([numerator, denominator], key) => [ dsgGroups[key].numerator > numerator ? dsgGroups[key].numerator : numerator, dsgGroups[key].denominator > denominator ? dsgGroups[key].denominator : denominator @@ -346,7 +347,7 @@ export const AddUpdate = ({ return null }} />, - indicator.measure === '1' ? + indicator.measure === measureType.UNIT ? 0 ? t('Total value') : t('Value') }} @@ -365,7 +366,7 @@ export const AddUpdate = ({ step={1} disabled={disableInputs} />, - (indicator.measure === '1' && period.updates.length > 0) && [ + (indicator.measure === measureType.UNIT && period.updates.length > 0) && [
      {t('Updated actual value')}
      ], - indicator.measure === '2' && [ + indicator.measure === measureType.PERCENTAGE && [ ]}
      - {!mneView && !(indicator.measure === '2' && period.updates.length > 0) && + {!mneView && !(indicator.measure === measureType.PERCENTAGE && period.updates.length > 0) && it.status === 'A' || it.status === 'R')[0]} {...{ period, indicator }} /> } {(mneView && indicator.type === 1) && ( diff --git a/akvo/rsr/spa/app/components/LineChart.jsx b/akvo/rsr/spa/app/components/LineChart.jsx index 0f35b442dc..3db282e529 100644 --- a/akvo/rsr/spa/app/components/LineChart.jsx +++ b/akvo/rsr/spa/app/components/LineChart.jsx @@ -249,7 +249,7 @@ const LineChart = ({ )} {points.map((p, px) => { - const [x, y] = p + const [cx, cy] = p return ( )} > - + ) })} diff --git a/akvo/rsr/spa/app/components/PrevUpdate.jsx b/akvo/rsr/spa/app/components/PrevUpdate.jsx index 290d6db7ad..c709c07c46 100644 --- a/akvo/rsr/spa/app/components/PrevUpdate.jsx +++ b/akvo/rsr/spa/app/components/PrevUpdate.jsx @@ -9,6 +9,7 @@ import { nicenum } from '../utils/misc' import statusPending from '../images/status-pending.svg' import statusApproved from '../images/status-approved.svg' import { AllSubmissionsModal } from './AllSubmissionsModal' +import { measureType } from '../utils/constants' export const PrevUpdate = ({ update, period, indicator }) => { const [showSubmissionsModal, setShowSubmissionsModal] = useState(false) @@ -42,7 +43,7 @@ export const PrevUpdate = ({ update, period, indicator }) => {
      ] : [
      - {indicator.measure === '1' && + {indicator.measure === measureType.UNIT &&
      {nicenum(update.value)} @@ -70,7 +71,7 @@ export const PrevUpdate = ({ update, period, indicator }) => { ])}
      } - {indicator.measure === '2' && + {indicator.measure === measureType.PERCENTAGE && [
      diff --git a/akvo/rsr/spa/app/modules/results-admin/ResultAdmin.jsx b/akvo/rsr/spa/app/modules/results-admin/ResultAdmin.jsx index a584f3ae91..90024bcada 100644 --- a/akvo/rsr/spa/app/modules/results-admin/ResultAdmin.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/ResultAdmin.jsx @@ -147,10 +147,9 @@ const ResultAdmin = ({ api .get(`/indicator_period_data_framework/${item.id}/`) .then(({ data }) => { - const { disaggregations, ...props } = data setEditing({ ...item, - ...props, + ...data, indicator, note: data?.comments[0]?.comment || '', period: indicator?.periods?.find((p) => p.id === item.period.id) diff --git a/akvo/rsr/spa/app/modules/results-admin/components/QuantitativeIndicator.jsx b/akvo/rsr/spa/app/modules/results-admin/components/QuantitativeIndicator.jsx index b16d69527e..d2b89d0f47 100644 --- a/akvo/rsr/spa/app/modules/results-admin/components/QuantitativeIndicator.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/components/QuantitativeIndicator.jsx @@ -2,6 +2,7 @@ import React from 'react' import { Form, Typography } from 'antd' import { useTranslation } from 'react-i18next' import { nicenum } from '../../../utils/misc' +import { measureType } from '../../../utils/constants' const { Text } = Typography @@ -14,7 +15,7 @@ const QuantitativeIndicator = ({ indicator, period, numerator, denominator, amou return (
      { - indicator.measure === '1' + indicator.measure === measureType.UNIT ? ( <> 0 ? t('Total value') : t('Value')}> diff --git a/akvo/rsr/spa/app/modules/results-admin/components/ReportedEdit.jsx b/akvo/rsr/spa/app/modules/results-admin/components/ReportedEdit.jsx index 59bb3bd1dd..50556986c4 100644 --- a/akvo/rsr/spa/app/modules/results-admin/components/ReportedEdit.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/components/ReportedEdit.jsx @@ -4,7 +4,6 @@ import React, { useState, useEffect } from 'react' import { Row, Col, Button, Typography, Icon } from 'antd' import { Form as FinalForm } from 'react-final-form' import { useTranslation } from 'react-i18next' -import SimpleMarkdown from 'simple-markdown' import axios from 'axios' import humps from 'humps' @@ -43,8 +42,6 @@ const ReportedEdit = ({ const { t } = useTranslation() const [submitting, setSubmitting] = useState(false) const [fileSet, setFileSet] = useState([]) - const mdParse = SimpleMarkdown.defaultBlockParse - const mdOutput = SimpleMarkdown.defaultOutput const submitStatus = editing?.status === 'P' ? 'P' : mneView ? 'A' : 'P' const disaggregations = [] diff --git a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx index 7fdc771e8a..c66a63b9fa 100644 --- a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx @@ -5,17 +5,22 @@ import { useTranslation } from 'react-i18next' import classNames from 'classnames' import SimpleMarkdown from 'simple-markdown' import moment from 'moment' -import orderBy from 'lodash/orderBy' +import { + groupBy, + orderBy, +} from 'lodash' + import DsgOverview from '../../results/dsg-overview' import { DeclinedStatus } from '../../../components/DeclinedStatus' import { PrevUpdate } from '../../../components/PrevUpdate' import { StatusUpdate } from '../../../components/StatusUpdate' import FinalField from '../../../utils/final-field' -import { nicenum } from '../../../utils/misc' +import { getMaxDisaggregation, getSumValues, nicenum } from '../../../utils/misc' import RTE from '../../../utils/rte' import ScoringField from '../../../components/ScoringField' import LineChart from '../../../components/LineChart' import { isNSOProject } from '../../../utils/feat-flags' +import { measureType } from '../../../utils/constants' const { Text } = Typography @@ -53,7 +58,7 @@ const ReportedForm = ({
      {group.name}
      {group.dimensionValues.map(dsg => { - return indicator.measure === '1' ? ( + return indicator.measure === measureType.UNIT ? ( it.typeId === dsg.id && group.id === it.groupId)}].value`} control="input-number" @@ -90,168 +95,180 @@ const ReportedForm = ({ )}
      )} - {indicator.type === 1 ? [ - <> - {init?.disaggregations?.length > 0 && ( - { - if (isNSOProject(project)) { + {indicator.type === 1 ? + ( + <> + {init?.disaggregations?.length > 0 && ( + { + if (isNSOProject(project)) { + return null + } + const _dsgGrouped = groupBy(input?.value, 'category') + const _dsgValues = Object + .values(_dsgGrouped) + ?.map((values) => values?.map((v) => ({ ...v, numerator: v?.numerator || null, denominator: v?.denominator || null }))) + ?.map((values) => ({ + value: getSumValues(values, 'value'), + numerator: getSumValues(values, 'numerator'), + denominator: getSumValues(values, 'denominator'), + })) + const dsgGroups = Object + .keys(_dsgGrouped) + ?.reduce((obj, key, index) => ({ + ...obj, + [key]: _dsgValues[index] + }), {}) + const categories = Object.keys(dsgGroups) + + if (categories.length > 0 && indicator.measure === measureType.UNIT) { + const value = getMaxDisaggregation(_dsgValues, 'value') + form.change('value', value) + } + if (categories.length > 0 && indicator.measure === measureType.PERCENTAGE) { + const numerator = getMaxDisaggregation(_dsgValues, 'numerator') + const denominator = getMaxDisaggregation(_dsgValues, 'denominator') + form.change('numerator', numerator) + form.change('denominator', denominator) + } return null - } - const dsgGroups = {} - if (input?.value?.length) { - input.value.forEach(item => { - if (!dsgGroups[item.category]) dsgGroups[item.category] = { value: 0, numerator: 0, denominator: 0 } - if (item.value) dsgGroups[item.category].value += item.value - if (item.numerator) dsgGroups[item.category].numerator += item.numerator - if (item.denominator) dsgGroups[item.category].denominator += item.denominator - }) - } - const categories = Object.keys(dsgGroups) - if (categories.length > 0 && indicator.measure === '1') { - const value = categories.reduce((acc, key) => dsgGroups[key].value > acc ? dsgGroups[key].value : acc, 0) - if (value > 0) form.change('value', value) - } - if (categories.length > 0 && indicator.measure === '2') { - const [numerator, denominator] = categories.reduce(([numerator, denominator], key) => [ - dsgGroups[key].numerator > numerator ? dsgGroups[key].numerator : numerator, - dsgGroups[key].denominator > denominator ? dsgGroups[key].denominator : denominator - ], [0, 0]) - if (numerator > 0) form.change('numerator', numerator) - if (denominator > 0) form.change('denominator', denominator) - } - return null - }} - /> - )} - , - indicator.measure === '1' ? - 0 ? t('Total value') : t('Value') }} - name="value" - control="input-number" - min={-Infinity} - step={1} - disabled={disableInputs} - /> : - , - (indicator.measure === '1' && period.updates.length > 0) && [ -
      -
      {t('Updated actual value')}
      - { - const updatedTotal = disableInputs ? 0 : (input.value > 0 ? input.value : 0) - return ( -
      - {nicenum(updatedTotal)} - {period.targetValue > 0 && {(Math.round((updatedTotal / period.targetValue) * 100 * 10) / 10)}% of target} + }} + /> + )} + {(indicator.measure === measureType.UNIT) && ( + <> + 0 ? t('Total value') : t('Value') }} + name="value" + control="input-number" + step={1} + disabled={disableInputs} + /> + {(period.updates.length > 0) && ( +
      +
      {t('Updated actual value')}
      + { + const updatedTotal = disableInputs ? 0 : input.value + return ( +
      + {nicenum(updatedTotal)} + {period.targetValue > 0 && {(Math.round((updatedTotal / period.targetValue) * 100 * 10) / 10)}% of target} +
      + ) + }} + />
      - ) - }} + )} + + )} + {(indicator.measure === measureType.PERCENTAGE) && ( +
      +
      + + +
      +
      + + {({ values }) => { + if (values.numerator !== '' && values.numerator != null && values.denominator) { + const value = Math.round((values.numerator / values.denominator) * 100 * 10) / 10 + if (value !== values.value) { + form.change('value', value) + } + return `${value}%` + } + return null + }} + +
      +
      + )} + + ) + : [ // qualitative indicator + indicator.scores?.length > 0 && ( + } /> -
      - ], - indicator.measure === '2' && [ - , -
      - - {({ values }) => { - if (values.numerator !== '' && values.numerator != null && values.denominator) { - const value = Math.round((values.numerator / values.denominator) * 100 * 10) / 10 - if (value !== values.value) { - form.change('value', value) - } - return `${value}%` - } - return null - }} - -
      - ] - ] : [ // qualitative indicator - indicator.scores?.length > 0 && ( + ), +
      {t('New update')}
      , } + name="narrative" + render={({ input }) => { + if (disableInputs) { + const parse = SimpleMarkdown.defaultBlockParse + const mdOutput = SimpleMarkdown.defaultOutput + return
      {mdOutput(parse(input.value))}
      + } + return [ + + ] + }} /> - ), -
      {t('New update')}
      , - { - if (disableInputs) { - const parse = SimpleMarkdown.defaultBlockParse - const mdOutput = SimpleMarkdown.defaultOutput - return
      {mdOutput(parse(input.value))}
      - } - return [ - - ] - }} - /> - ]} + ]}
      - {!mneView && !(indicator.measure === '2' && period.updates.length > 0) && + {!mneView && !(indicator.measure === measureType.PERCENTAGE && period.updates.length > 0) && it.status === 'A' || it.status === 'R')[0]} {...{ period, indicator }} /> } - {(mneView && indicator.type === 1) && ( - disaggregations.length > 0 ? - ( - - {({ values }) => { - const periodUpdates = [...period.updates, { ...values, status: 'D' }] - const dgs = [...periodUpdates.reduce((acc, val) => [...acc, ...val.disaggregations.map(it => ({ ...it, status: val.status }))], [])] - if (dgs.length) { - disaggregations = dgs.map((dg, dgx) => ({ ...dg, typeId: disaggregations[dgx]?.typeId })) - } - const valueUpdates = periodUpdates.map(it => ({ value: it.value, status: it.status })) - return { editPeriod(props, indicator) }, values: valueUpdates }} /> - }} - - ) : -
      - - {({ values }) => { - const updates = [...period.updates, { ...values, status: 'D' }].filter(it => it !== null) - let data = updates?.map(u => ({ - label: u.createdAt ? moment(u.createdAt, 'YYYY-MM-DD').format('DD-MM-YYYY') : null, - unix: u.createdAt ? moment(u.createdAt, 'YYYY-MM-DD').unix() : null, - y: u.value || 0 - })) - data = orderBy(data, ['unix'], ['asc']).map((u, index) => ({ ...u, x: index })) - return ( - - ) - }} - -
      + {(mneView && indicator.type === 1 && disaggregations.length > 0) && ( + + {({ values }) => { + const periodUpdates = [...period.updates, { ...values, status: 'D' }] + const dgs = [...periodUpdates.reduce((acc, val) => [...acc, ...val.disaggregations.map(it => ({ ...it, status: val.status }))], [])] + if (dgs.length) { + disaggregations = dgs.map((dg, dgx) => ({ ...dg, typeId: disaggregations[dgx]?.typeId })) + } + const valueUpdates = periodUpdates.map(it => ({ value: it.value, status: it.status })) + return { editPeriod(props, indicator) }, values: valueUpdates }} /> + }} + + )} + {(indicator?.measure === measureType.UNIT && indicator?.type === 1) && ( +
      + + {({ values }) => { + const updates = [...period.updates, { ...values, status: 'D' }].filter(it => it !== null) + let data = updates?.map(u => ({ + label: u.createdAt ? moment(u.createdAt, 'YYYY-MM-DD').format('DD-MM-YYYY') : null, + unix: u.createdAt ? moment(u.createdAt, 'YYYY-MM-DD').unix() : null, + y: u.value || 0 + })) + data = orderBy(data, ['unix'], ['asc']).map((u, index) => ({ ...u, x: index })) + return ( + + ) + }} + +
      )}
      diff --git a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.scss b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.scss index 7e1243d184..7cc2693a0f 100644 --- a/akvo/rsr/spa/app/modules/results-overview/ResultOverview.scss +++ b/akvo/rsr/spa/app/modules/results-overview/ResultOverview.scss @@ -262,15 +262,15 @@ width: 170px; font-weight: 600; } - .perc { - color: #4e998e; - font-size: 33px; - font-weight: 600; - position: absolute; - left: 53%; - width: calc(100% - 200px); - top: 10px; - text-align: center; + .percentage-indicator { + display: flex; + gap: 16px; + align-items: center; + .perc { + color: #4e998e; + font-size: 33px; + font-weight: 600; + } } } .prev-value-holder { diff --git a/akvo/rsr/spa/app/modules/results-overview/components/UpdateItems.jsx b/akvo/rsr/spa/app/modules/results-overview/components/UpdateItems.jsx index fc6e084d67..16dd7cdf94 100644 --- a/akvo/rsr/spa/app/modules/results-overview/components/UpdateItems.jsx +++ b/akvo/rsr/spa/app/modules/results-overview/components/UpdateItems.jsx @@ -12,6 +12,7 @@ import { StatusPeriod } from '../../../components/StatusPeriod' import editButton from '../../../images/edit-button.svg' import ProgressBar from '../../../components/ProgressBar' import LineChart from '../../../components/LineChart' +import { measureType } from '../../../utils/constants' const { Panel } = Collapse const Aux = node => node.children @@ -133,7 +134,7 @@ const UpdateItems = ({ {indicator.type === 1 &&
      {String(update.value).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} - {indicator.measure === '2' && %} + {indicator.measure === measureType.PERCENTAGE && %}
      }
      diff --git a/akvo/rsr/spa/app/modules/results/EnumeratorPage.jsx b/akvo/rsr/spa/app/modules/results/EnumeratorPage.jsx index d0325ba2a5..0e498916e7 100644 --- a/akvo/rsr/spa/app/modules/results/EnumeratorPage.jsx +++ b/akvo/rsr/spa/app/modules/results/EnumeratorPage.jsx @@ -29,6 +29,7 @@ import { FilterBar } from '../results-overview/components' import ReportedEdit from '../results-admin/components/ReportedEdit' import StatusIndicator from '../../components/StatusIndicator' import * as actions from './actions' +import { measureType } from '../../utils/constants' const { Text } = Typography @@ -102,7 +103,7 @@ const EnumeratorPage = ({ const myUpdates = p.updates.filter((u) => u?.userDetails?.id === userRdr.id) return ( (p?.canAddUpdate && p.indicator.measure !== '2') || - (p.indicator.measure === '2' && (p?.canAddUpdate || myUpdates.length)) + (p.indicator.measure === measureType.PERCENTAGE && (p?.canAddUpdate || myUpdates.length)) ) }) ?.filter((p) => { diff --git a/akvo/rsr/spa/app/modules/results/edit-update.jsx b/akvo/rsr/spa/app/modules/results/edit-update.jsx index c1722b4ce8..cc1a8fd461 100644 --- a/akvo/rsr/spa/app/modules/results/edit-update.jsx +++ b/akvo/rsr/spa/app/modules/results/edit-update.jsx @@ -4,6 +4,7 @@ import { useTranslation } from 'react-i18next' import classNames from 'classnames' import './edit-update.scss' import ScoringField from '../../components/ScoringField' +import { measureType } from '../../utils/constants' const { Item } = Form const inputNumberFormatting = { @@ -86,7 +87,7 @@ const EditUpdate = ({ update, handleUpdateEdit, indicator }) => { const value = dsgIndex > -1 ? update.disaggregations[dsgIndex].value : '' const numerator = dsgIndex > -1 ? update.disaggregations[dsgIndex].numerator : '' const denominator = dsgIndex > -1 ? update.disaggregations[dsgIndex].denominator : '' - return indicator.measure === '1' ? ( + return indicator.measure === measureType.UNIT ? ( String(val).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} @@ -120,7 +121,7 @@ const EditUpdate = ({ update, handleUpdateEdit, indicator }) => { )}
      )} - {indicator.measure === '1' && + {indicator.measure === measureType.UNIT && { /> } - {indicator.measure === '2' && [ + {indicator.measure === measureType.PERCENTAGE && [ node.children @@ -136,7 +137,7 @@ const Period = ({ setResults, period, measure, treeFilter, statusFilter, increas status } payload.scoreIndices = scoreIndices - if (indicator.measure === '2') { + if (indicator.measure === measureType.PERCENTAGE) { payload.numerator = sortedUpdates[editing].numerator payload.denominator = sortedUpdates[editing].denominator } @@ -306,7 +307,7 @@ const Period = ({ setResults, period, measure, treeFilter, statusFilter, increas
      {t('baseline value')}
      -
      {baseline.value}{indicator.measure === '2' && %}
      +
      {baseline.value}{indicator.measure === measureType.PERCENTAGE && %}
      {t('baseline year')}
      @@ -346,7 +347,7 @@ const Period = ({ setResults, period, measure, treeFilter, statusFilter, increas { indicator.type === 1 && editing !== index &&
      - {String(update.value).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} {indicator.measure === '2' && %} + {String(update.value).replace(/\B(?=(\d{3})+(?!\d))/g, ',')} {indicator.measure === measureType.PERCENTAGE && %}
      }
      diff --git a/akvo/rsr/spa/app/modules/results/update.jsx b/akvo/rsr/spa/app/modules/results/update.jsx index f2630fba85..e3e17d10a8 100644 --- a/akvo/rsr/spa/app/modules/results/update.jsx +++ b/akvo/rsr/spa/app/modules/results/update.jsx @@ -8,6 +8,7 @@ import 'react-image-lightbox/style.css' import api from '../../utils/api' import { AuditTrail } from '../../components/AuditTrail' import { setNumberFormat } from '../../utils/misc' +import { measureType } from '../../utils/constants' const { Panel } = Collapse const { Text } = Typography @@ -101,7 +102,7 @@ const Update = ({ update, period, indicator, fullUpdates, setFullUpdates }) => { ) } - {indicator.measure === '2' && [ + {indicator.measure === measureType.PERCENTAGE && [
      {update.numerator ? (
      diff --git a/akvo/rsr/spa/app/utils/constants.js b/akvo/rsr/spa/app/utils/constants.js index f5787bd248..70ec30177a 100644 --- a/akvo/rsr/spa/app/utils/constants.js +++ b/akvo/rsr/spa/app/utils/constants.js @@ -32,3 +32,8 @@ export const indicatorTypes = [ { label: 'quantitative', value: 1}, { label: 'qualitative', value: 2} ] + +export const measureType = { + UNIT: '1', + PERCENTAGE: '2', +} diff --git a/akvo/rsr/spa/app/utils/final-field.jsx b/akvo/rsr/spa/app/utils/final-field.jsx index 180bda4034..96a22f16aa 100644 --- a/akvo/rsr/spa/app/utils/final-field.jsx +++ b/akvo/rsr/spa/app/utils/final-field.jsx @@ -40,7 +40,7 @@ const CONTROLS = { return }, 'input-number': ({ input, meta, control, currencySymbol, ...props}) => { - return { if (validateNumber(val)) input.onChange(val); else if(val === '') input.onChange(undefined) }, ...inputNumberAmountFormatting(currencySymbol), min: 1, ...props}} /> + return { input.onChange(val) }, ...inputNumberAmountFormatting(currencySymbol), ...props}} /> }, textarea: ({ input, meta, control, ...props }) => , select: ({options, input, meta, control, withEmptyOption, withValuePrefix, ...props}) => { diff --git a/akvo/rsr/spa/app/utils/misc.js b/akvo/rsr/spa/app/utils/misc.js index d101fa0403..4fc78ca698 100644 --- a/akvo/rsr/spa/app/utils/misc.js +++ b/akvo/rsr/spa/app/utils/misc.js @@ -1,5 +1,6 @@ /* globals FileReader, window */ import { diff } from 'deep-object-diff' +import sumBy from 'lodash/sumBy' export const datePickerConfig = { format: 'DD/MM/YYYY', @@ -188,3 +189,15 @@ export const wordWrap = (s, w) => { } export const splitPeriod = value => value?.split('-')?.map((v) => v.trim()) + +export const getSumValues = (values, field) => { + const isNull = values?.filter((v) => v[field] === null)?.length === values?.length + return isNull ? null : sumBy(values, field) +} + +export const getMaxDisaggregation = (values, field) => { + const allValues = values + ?.filter((d) => d[field] !== null) + ?.map((d) => d[field]) + return allValues?.length ? Math.max(...allValues) : null +} From da27e638de015656c258060b15b19eef6fdf2e58 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Wed, 23 Nov 2022 20:11:59 +0700 Subject: [PATCH 49/59] [#5167] Create an endpoint to make django-lockdown useable --- akvo/rsr/views/__init__.py | 7 +++++++ akvo/urls.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/akvo/rsr/views/__init__.py b/akvo/rsr/views/__init__.py index ac1ca8c76b..fedecfb695 100644 --- a/akvo/rsr/views/__init__.py +++ b/akvo/rsr/views/__init__.py @@ -14,3 +14,10 @@ def index(request): """Redirect user to project directory or My RSR.""" return HttpResponseRedirect(reverse('project-directory', args=[])) + + +def lockpass(request): + next_page = request.GET.get("next") + return HttpResponseRedirect( + next_page if next_page else reverse("project-directory", args=[]) + ) diff --git a/akvo/urls.py b/akvo/urls.py index 61d562893b..3cb586e8d9 100644 --- a/akvo/urls.py +++ b/akvo/urls.py @@ -46,6 +46,10 @@ url(r'^$', views.index, name='index'), + # Hack to prompt password on password-protected partner sites + url(r'^lockpass/$', + views.lockpass, name='lockpass'), + # Projects url(r'^projects/$', project.directory, name='project-directory'), From 7b4691151297804cca17e1b987fbf0d11db71b2f Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Thu, 24 Nov 2022 12:20:11 +0700 Subject: [PATCH 50/59] [#5167] FE implementation for lock pass endpoint - Handle password protection for project-directory (landing) page. - Handle password protection for public project page. --- akvo/rsr/dir/app/modules/index/view.jsx | 5 ++++- akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/akvo/rsr/dir/app/modules/index/view.jsx b/akvo/rsr/dir/app/modules/index/view.jsx index 2a9201a048..681d1a53b0 100644 --- a/akvo/rsr/dir/app/modules/index/view.jsx +++ b/akvo/rsr/dir/app/modules/index/view.jsx @@ -83,7 +83,10 @@ const View = () => { setFilters(defaults) } } - }, [apiData]) + if (apiError && apiError.response && apiError.response.status === 403) { + window.location.href = '/en/lockpass/?next=' + } + }, [apiData, apiError]) useEffect(() => { document.getElementById('root').classList.add(window.location.host.split('.')[0]) }, []) diff --git a/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx b/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx index f9b07420c1..72e387e7ee 100644 --- a/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx +++ b/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx @@ -1,4 +1,4 @@ -/* global document */ +/* global window, document */ import React, { useEffect, useState } from 'react' import { Menu } from 'antd' import { Switch, Route, useHistory } from 'react-router-dom' @@ -84,6 +84,9 @@ const ProjectPage = ({ match: { params }, location }) => { } if ((loading && (apiError || projectError)) || (loading && user && !apiError)) { setLoading(false) + if (projectError && projectError.response && projectError.response.status === 403) { + window.location.href = `/en/lockpass/?next=${window.location.href}` + } } // eslint-disable-next-line no-restricted-globals if (!isNaN(currentPath) && menu !== HOME_KEY) { From 1c89f66c997e66b7332ab71879eb1737b6887f29 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Thu, 24 Nov 2022 15:44:36 +0700 Subject: [PATCH 51/59] [#5167] Set 403 redirect only for partner sites --- akvo/rsr/dir/app/modules/index/view.jsx | 3 ++- akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx | 3 ++- akvo/rsr/dir/app/root.jsx | 3 ++- akvo/rsr/dir/app/utils/misc.js | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/akvo/rsr/dir/app/modules/index/view.jsx b/akvo/rsr/dir/app/modules/index/view.jsx index 681d1a53b0..6cbd4acc00 100644 --- a/akvo/rsr/dir/app/modules/index/view.jsx +++ b/akvo/rsr/dir/app/modules/index/view.jsx @@ -11,6 +11,7 @@ import Map, { projectsToFeatureData } from './map' import Search from './search' import FilterBar from './filter-bar' import api from '../../utils/api' +import { isPartnerSites } from '../../utils/misc' const isLocal = window.location.href.indexOf('localhost') !== -1 || window.location.href.indexOf('localakvoapp') !== -1 const urlPrefix = isLocal ? 'http://rsr.akvo.org' : '' @@ -83,7 +84,7 @@ const View = () => { setFilters(defaults) } } - if (apiError && apiError.response && apiError.response.status === 403) { + if ((apiError && apiError.response && apiError.response.status === 403) && isPartnerSites()) { window.location.href = '/en/lockpass/?next=' } }, [apiData, apiError]) diff --git a/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx b/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx index 72e387e7ee..5e7f3ecbfa 100644 --- a/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx +++ b/akvo/rsr/dir/app/modules/project-page/ProjectPage.jsx @@ -28,6 +28,7 @@ import { UPDATES_KEY, projectPath, } from '../../utils/config' +import { isPartnerSites } from '../../utils/misc' const ProjectPage = ({ match: { params }, location }) => { @@ -84,7 +85,7 @@ const ProjectPage = ({ match: { params }, location }) => { } if ((loading && (apiError || projectError)) || (loading && user && !apiError)) { setLoading(false) - if (projectError && projectError.response && projectError.response.status === 403) { + if ((projectError && projectError.response && projectError.response.status === 403) && isPartnerSites()) { window.location.href = `/en/lockpass/?next=${window.location.href}` } } diff --git a/akvo/rsr/dir/app/root.jsx b/akvo/rsr/dir/app/root.jsx index ae85956758..b8f18cff77 100644 --- a/akvo/rsr/dir/app/root.jsx +++ b/akvo/rsr/dir/app/root.jsx @@ -8,12 +8,13 @@ import WcaroRouter from './modules/wcaro-index/router' import ProjectPage from './modules/project-page/ProjectPage' import scheduleDemo from './modules/schedule-demo' import { Home } from './modules/home' +import { isPartnerSites } from './utils/misc' export default () => { const isUNEP = window.location.href.indexOf('//unep.') !== -1 // eslint-disable-next-line no-unused-vars const isWcaro = window.location.href.indexOf('//wcaro.') !== -1 - const isPartner = window.location.href.indexOf('localakvoapp') !== -1 || window.location.href.indexOf('akvoapp') !== -1 + const isPartner = isPartnerSites() return ( diff --git a/akvo/rsr/dir/app/utils/misc.js b/akvo/rsr/dir/app/utils/misc.js index b5ff9e4b05..f06d0987d1 100644 --- a/akvo/rsr/dir/app/utils/misc.js +++ b/akvo/rsr/dir/app/utils/misc.js @@ -1,4 +1,5 @@ /* eslint-disable no-useless-escape */ +/* global window */ import moment from 'moment' import chunk from 'lodash/chunk' import orderBy from 'lodash/orderBy' @@ -99,3 +100,5 @@ export const getLogo = logo => logo export const getFirstPhoto = photos => (photos && photos.length) ? photos.slice(0, 1).pop() : null + +export const isPartnerSites = () => (window.location.href.indexOf('localakvoapp') !== -1 || window.location.href.indexOf('akvoapp') !== -1) From 670633d286affe7c37f07174babd3946fe147c77 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Thu, 24 Nov 2022 15:47:27 +0700 Subject: [PATCH 52/59] [#5167] Add documentation on lockpass view handler --- akvo/rsr/views/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/akvo/rsr/views/__init__.py b/akvo/rsr/views/__init__.py index fedecfb695..365d32f763 100644 --- a/akvo/rsr/views/__init__.py +++ b/akvo/rsr/views/__init__.py @@ -17,6 +17,14 @@ def index(request): def lockpass(request): + """Endpoint to make the password-protected partner site useable for users. + + This is a hack that utilizes the Django-lockdown mechanism to prompt the password page + for password-protected partner sites. + + See: akvo.rsr.middleware.RSRLockdownMiddleware + """ + next_page = request.GET.get("next") return HttpResponseRedirect( next_page if next_page else reverse("project-directory", args=[]) From b94a0e55770806da8a9c239852737dc839c2c267 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Fri, 25 Nov 2022 09:55:32 +0700 Subject: [PATCH 53/59] [#5167] Replace location.href with .hostname to check partner sites --- akvo/rsr/dir/app/utils/misc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/akvo/rsr/dir/app/utils/misc.js b/akvo/rsr/dir/app/utils/misc.js index f06d0987d1..1b45de6825 100644 --- a/akvo/rsr/dir/app/utils/misc.js +++ b/akvo/rsr/dir/app/utils/misc.js @@ -101,4 +101,4 @@ export const getFirstPhoto = photos => (photos && photos.length) ? photos.slice(0, 1).pop() : null -export const isPartnerSites = () => (window.location.href.indexOf('localakvoapp') !== -1 || window.location.href.indexOf('akvoapp') !== -1) +export const isPartnerSites = () => (window.location.hostname.endsWith('localakvoapp.org') || window.location.hostname.endsWith('akvoapp.org')) From 2c36015613e6b07dc8c21172600a1c278ade9c65 Mon Sep 17 00:00:00 2001 From: ifirmawan Date: Mon, 12 Dec 2022 09:42:27 +0700 Subject: [PATCH 54/59] [#5161] Fix whitescreen when attempting a negative value for dsg Because a negative value is led by a dash/minus symbol which is the data type of a string. Then we need to validate isNaN before we put it into the value field. --- .../app/modules/results-admin/components/ReportedForm.jsx | 4 +++- akvo/rsr/spa/app/utils/final-field.jsx | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx index c66a63b9fa..91094ed39d 100644 --- a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx @@ -124,7 +124,9 @@ const ReportedForm = ({ if (categories.length > 0 && indicator.measure === measureType.UNIT) { const value = getMaxDisaggregation(_dsgValues, 'value') - form.change('value', value) + if (!Number.isNaN(value)) { + form.change('value', value) + } } if (categories.length > 0 && indicator.measure === measureType.PERCENTAGE) { const numerator = getMaxDisaggregation(_dsgValues, 'numerator') diff --git a/akvo/rsr/spa/app/utils/final-field.jsx b/akvo/rsr/spa/app/utils/final-field.jsx index 96a22f16aa..aff45f1cad 100644 --- a/akvo/rsr/spa/app/utils/final-field.jsx +++ b/akvo/rsr/spa/app/utils/final-field.jsx @@ -39,9 +39,9 @@ const CONTROLS = { input: ({ input, meta, control, ...props }) => { return }, - 'input-number': ({ input, meta, control, currencySymbol, ...props}) => { - return { input.onChange(val) }, ...inputNumberAmountFormatting(currencySymbol), ...props}} /> - }, + 'input-number': ({ input, meta, control, currencySymbol, ...props}) => ( + + ), textarea: ({ input, meta, control, ...props }) => , select: ({options, input, meta, control, withEmptyOption, withValuePrefix, ...props}) => { return ( From 7afe790fade3aa01b8229a706b337f8b0d90332f Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Dec 2022 09:45:27 +0100 Subject: [PATCH 55/59] refactor: Reorganize migrations Production has priority so the migrations from master have to come after --- .../migrations/{0221_project_tree.py => 0223_project_tree.py} | 2 +- .../{0222_rm_iati_import.py => 0224_rm_iati_import.py} | 2 +- ...y => 0225_remove_indicatorperioddata_period_actual_value.py} | 2 +- ...224_indicator_cumulative.py => 0226_indicator_cumulative.py} | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename akvo/rsr/migrations/{0221_project_tree.py => 0223_project_tree.py} (98%) rename akvo/rsr/migrations/{0222_rm_iati_import.py => 0224_rm_iati_import.py} (97%) rename akvo/rsr/migrations/{0223_remove_indicatorperioddata_period_actual_value.py => 0225_remove_indicatorperioddata_period_actual_value.py} (88%) rename akvo/rsr/migrations/{0224_indicator_cumulative.py => 0226_indicator_cumulative.py} (89%) diff --git a/akvo/rsr/migrations/0221_project_tree.py b/akvo/rsr/migrations/0223_project_tree.py similarity index 98% rename from akvo/rsr/migrations/0221_project_tree.py rename to akvo/rsr/migrations/0223_project_tree.py index 9df02682ff..20c13e4afb 100644 --- a/akvo/rsr/migrations/0221_project_tree.py +++ b/akvo/rsr/migrations/0223_project_tree.py @@ -34,7 +34,7 @@ def migrate_contributing_projects(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('rsr', '0220_emailreportjob'), + ('rsr', '0222_indicatorperiodaggregationjob_root_period'), ] operations = [ diff --git a/akvo/rsr/migrations/0222_rm_iati_import.py b/akvo/rsr/migrations/0224_rm_iati_import.py similarity index 97% rename from akvo/rsr/migrations/0222_rm_iati_import.py rename to akvo/rsr/migrations/0224_rm_iati_import.py index 3fe5cd3168..5bf7aa2008 100644 --- a/akvo/rsr/migrations/0222_rm_iati_import.py +++ b/akvo/rsr/migrations/0224_rm_iati_import.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('rsr', '0221_project_tree'), + ('rsr', '0223_project_tree'), ] operations = [ diff --git a/akvo/rsr/migrations/0223_remove_indicatorperioddata_period_actual_value.py b/akvo/rsr/migrations/0225_remove_indicatorperioddata_period_actual_value.py similarity index 88% rename from akvo/rsr/migrations/0223_remove_indicatorperioddata_period_actual_value.py rename to akvo/rsr/migrations/0225_remove_indicatorperioddata_period_actual_value.py index 03cd089cd5..850b7f450e 100644 --- a/akvo/rsr/migrations/0223_remove_indicatorperioddata_period_actual_value.py +++ b/akvo/rsr/migrations/0225_remove_indicatorperioddata_period_actual_value.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('rsr', '0222_rm_iati_import'), + ('rsr', '0224_rm_iati_import'), ] operations = [ diff --git a/akvo/rsr/migrations/0224_indicator_cumulative.py b/akvo/rsr/migrations/0226_indicator_cumulative.py similarity index 89% rename from akvo/rsr/migrations/0224_indicator_cumulative.py rename to akvo/rsr/migrations/0226_indicator_cumulative.py index 92dcf0e1a4..9f2ae19bb4 100644 --- a/akvo/rsr/migrations/0224_indicator_cumulative.py +++ b/akvo/rsr/migrations/0226_indicator_cumulative.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('rsr', '0223_remove_indicatorperioddata_period_actual_value'), + ('rsr', '0225_remove_indicatorperioddata_period_actual_value'), ] operations = [ From 13295bad60a1f4bc9d302f61f4cf7423e6888eca Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Dec 2022 11:38:18 +0100 Subject: [PATCH 56/59] refactor: Fix import order Models weren't initialized yet due to a circular import --- akvo/rsr/models/result/indicator_period_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index b25af165e9..a3eaa05c88 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -21,7 +21,6 @@ from akvo.rsr.fields import ValidXMLCharField, ValidXMLTextField from akvo.rsr.mixins import TimestampsMixin, IndicatorUpdateMixin from akvo.utils import rsr_image_path -from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job class IndicatorPeriodData(TimestampsMixin, IndicatorUpdateMixin, models.Model): @@ -101,6 +100,7 @@ def save(self, recalculate=True, *args, **kwargs): # In case the status is approved, recalculate the period if recalculate and self.status == self.STATUS_APPROVED_CODE: # FIXME: Should we call this even when status is not approved? + from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job schedule_aggregation_job(self.period) self.period.update_actual_comment() # Update score even when the update is not approved, yet. It handles the @@ -115,6 +115,7 @@ def delete(self, *args, **kwargs): # In case the status was approved, recalculate the period if old_status == self.STATUS_APPROVED_CODE: + from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job schedule_aggregation_job(period) self.period.update_actual_comment() self.period.update_score() From fef351c009085540a12af7dea53f27796f841c98 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Mon, 26 Dec 2022 17:14:32 +0700 Subject: [PATCH 57/59] refactor: Fix cumulative reporting aggregation --- .../test_cumulative_updates.py | 11 +++++++++++ .../views/py_reports/test_cumulative_updates.py | 8 ++++++++ akvo/rsr/usecases/period_update_aggregation.py | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/akvo/rsr/tests/results_framework/test_cumulative_updates.py b/akvo/rsr/tests/results_framework/test_cumulative_updates.py index 31520e5a47..0e32b7485f 100644 --- a/akvo/rsr/tests/results_framework/test_cumulative_updates.py +++ b/akvo/rsr/tests/results_framework/test_cumulative_updates.py @@ -3,6 +3,7 @@ from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.tests.utils import ProjectFixtureBuilder from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE +from akvo.rsr.usecases.period_update_aggregation import aggregate class CumulativeTestMixin: @@ -54,6 +55,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(self.period1.object) self.period2 = self.project.get_period(period_start=self.PERIOD_2_START) self.period2.add_update(user=user, value=3, disaggregations={ @@ -68,6 +70,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(self.period2.object) def test_period1(self): period1 = self.project.periods.get(id=self.period1.id) @@ -107,6 +110,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(self.period1.object) self.period2 = self.project.get_period(period_start=self.PERIOD_2_START) self.period2.add_update(user=user2, value=3, disaggregations={ @@ -121,6 +125,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(self.period2.object) def test_period1(self): period1 = self.project.periods.get(id=self.period1.id) @@ -154,6 +159,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_1: {'numerator': 1, 'denominator': 4}, } }) + aggregate(self.period1.object) self.period2 = self.project.get_period(period_start=self.PERIOD_2_START) self.period2.add_update(user=user2, numerator=2, denominator=4, disaggregations={ @@ -162,6 +168,7 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'numerator': 1, 'denominator': 4}, } }) + aggregate(self.period2.object) def test_period1(self): period1 = self.project.periods.get(id=self.period1.id) @@ -206,12 +213,14 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 0}, } }) + aggregate(contrib1_period1.object) contrib1_period2.add_update(user=user2, value=1, disaggregations={ self.DISAGGREGATION_CATEGORY: { self.DISAGGREGATION_TYPE_1: {'value': 0}, self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib1_period2.object) contrib2_period1.add_update(user=user1, value=1, disaggregations={ self.DISAGGREGATION_CATEGORY: { @@ -219,12 +228,14 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib2_period1.object) contrib2_period2.add_update(user=user1, value=2, disaggregations={ self.DISAGGREGATION_CATEGORY: { self.DISAGGREGATION_TYPE_1: {'value': 1}, self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib2_period2.object) def test_lead_period1(self): lead_period1 = self.lead_project.periods.get(period_start=self.PERIOD_1_START) diff --git a/akvo/rsr/tests/views/py_reports/test_cumulative_updates.py b/akvo/rsr/tests/views/py_reports/test_cumulative_updates.py index 6597c0d35b..b51b4cd149 100644 --- a/akvo/rsr/tests/views/py_reports/test_cumulative_updates.py +++ b/akvo/rsr/tests/views/py_reports/test_cumulative_updates.py @@ -4,6 +4,7 @@ from akvo.rsr.models import Partnership, Project from akvo.rsr.tests.base import BaseTestCase from akvo.rsr.tests.utils import ProjectFixtureBuilder +from akvo.rsr.usecases.period_update_aggregation import aggregate from akvo.rsr.views.py_reports import ( results_indicators_with_map_pdf_reports, results_indicators_excel_report, @@ -76,6 +77,7 @@ def populate_project_updates_data(self, project): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(period1.object) period2 = project.get_period(period_start=self.PERIOD_2_START) period2.add_update(user=user2, value=3, disaggregations={ @@ -90,6 +92,7 @@ def populate_project_updates_data(self, project): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(period2.object) period3 = project.get_period(period_start=self.PERIOD_3_START) period3.add_update(user=user1, value=3, disaggregations={ @@ -104,6 +107,7 @@ def populate_project_updates_data(self, project): self.DISAGGREGATION_TYPE_2: {'value': 2}, } }) + aggregate(period3.object) class ObjectReaderCumulativeUpdateBaseTestCase(CumulativeTestMixin, ABC): @@ -227,12 +231,14 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 0}, } }) + aggregate(contrib1_period1.object) contrib1_period2.add_update(user=user2, value=1, disaggregations={ self.DISAGGREGATION_CATEGORY: { self.DISAGGREGATION_TYPE_1: {'value': 0}, self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib1_period2.object) contrib2_period1.add_update(user=user1, value=1, disaggregations={ self.DISAGGREGATION_CATEGORY: { @@ -240,12 +246,14 @@ def setUp(self): self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib2_period1.object) contrib2_period2.add_update(user=user1, value=2, disaggregations={ self.DISAGGREGATION_CATEGORY: { self.DISAGGREGATION_TYPE_1: {'value': 1}, self.DISAGGREGATION_TYPE_2: {'value': 1}, } }) + aggregate(contrib2_period2.object) self.result = program_overview_excel_report.get_results_framework(self.lead_project.object)[0] def get_period_contributor(self, period, project_title): diff --git a/akvo/rsr/usecases/period_update_aggregation.py b/akvo/rsr/usecases/period_update_aggregation.py index 0fdabba8db..6ff79cf617 100644 --- a/akvo/rsr/usecases/period_update_aggregation.py +++ b/akvo/rsr/usecases/period_update_aggregation.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from akvo.rsr.models import IndicatorPeriod -from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE, calculate_percentage +from akvo.rsr.models.result.utils import PERCENTAGE_MEASURE, calculate_percentage, get_per_user_latest_indicator_update_ids from akvo.rsr.models.result.disaggregation_aggregation import DisaggregationAggregation @@ -60,6 +60,21 @@ def _aggregate_disaggregation(period: IndicatorPeriod): def sum_updates(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]: + return sum_cumulative_updates(period) if period.indicator.is_cumulative() else sum_non_cumulative_updates(period) + + +def sum_cumulative_updates(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]: + ''' + This method assumes the user will submit cumulative updates in chronological order as it should. + ''' + IndicatorPeriodData = apps.get_model('rsr', 'IndicatorPeriodData') + latest_per_users = get_per_user_latest_indicator_update_ids(period) + value = IndicatorPeriodData.objects.filter(id__in=latest_per_users)\ + .aggregate(value=Sum('value'))['value'] + return value, None, None + + +def sum_non_cumulative_updates(period: IndicatorPeriod) -> Tuple[Optional[Decimal], Optional[Decimal], Optional[Decimal]]: result = period.approved_updates.aggregate(value=Sum('value'), numerator=Sum('numerator'), denominator=Sum('denominator')) return (result[k] for k in ('value', 'numerator', 'denominator')) From feee1007378ae6a62fefac8a370644fad59282f5 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Mon, 26 Dec 2022 17:44:57 +0700 Subject: [PATCH 58/59] refactor: Fix linting errors --- akvo/rsr/models/project.py | 3 +-- akvo/rsr/models/result/disaggregation_aggregation.py | 2 +- akvo/rsr/models/result/indicator_period.py | 2 +- akvo/rsr/models/result/indicator_period_data.py | 3 +-- .../spa/app/modules/results-admin/components/ReportedForm.jsx | 1 - 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/akvo/rsr/models/project.py b/akvo/rsr/models/project.py index daac4027a8..859a803932 100644 --- a/akvo/rsr/models/project.py +++ b/akvo/rsr/models/project.py @@ -6,9 +6,8 @@ """ import dataclasses import logging -from decimal import Decimal, InvalidOperation +from decimal import Decimal from typing import Dict, Generic, Hashable, Optional, TypeVar -import itertools import urllib.parse from django.conf import settings diff --git a/akvo/rsr/models/result/disaggregation_aggregation.py b/akvo/rsr/models/result/disaggregation_aggregation.py index 03276b8a43..1ce1ec00e8 100644 --- a/akvo/rsr/models/result/disaggregation_aggregation.py +++ b/akvo/rsr/models/result/disaggregation_aggregation.py @@ -9,7 +9,6 @@ from akvo.utils import ensure_decimal -from .indicator_period_data import IndicatorPeriodData from .utils import get_per_user_latest_indicator_update_ids @@ -53,6 +52,7 @@ def _get_local_cumulative_values(self, period, dimension_value): ) def _get_local_non_cumulative_values(self, period, dimension_value): + IndicatorPeriodData = apps.get_model('rsr', 'IndicatorPeriodData') return self.disaggregations.filter( update__period=period, update__status=IndicatorPeriodData.STATUS_APPROVED_CODE, diff --git a/akvo/rsr/models/result/indicator_period.py b/akvo/rsr/models/result/indicator_period.py index b4a493fda5..86fc533299 100644 --- a/akvo/rsr/models/result/indicator_period.py +++ b/akvo/rsr/models/result/indicator_period.py @@ -12,7 +12,7 @@ from django.utils.translation import ugettext_lazy as _ from .indicator_period_data import IndicatorPeriodData -from .utils import calculate_percentage, PERCENTAGE_MEASURE, QUALITATIVE, get_per_user_latest_indicator_update_ids +from .utils import calculate_percentage, PERCENTAGE_MEASURE, QUALITATIVE from akvo.rsr.fields import ValidXMLCharField, ValidXMLTextField diff --git a/akvo/rsr/models/result/indicator_period_data.py b/akvo/rsr/models/result/indicator_period_data.py index a3eaa05c88..15060cc134 100644 --- a/akvo/rsr/models/result/indicator_period_data.py +++ b/akvo/rsr/models/result/indicator_period_data.py @@ -20,6 +20,7 @@ QUANTITATIVE) from akvo.rsr.fields import ValidXMLCharField, ValidXMLTextField from akvo.rsr.mixins import TimestampsMixin, IndicatorUpdateMixin +from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job from akvo.utils import rsr_image_path @@ -100,7 +101,6 @@ def save(self, recalculate=True, *args, **kwargs): # In case the status is approved, recalculate the period if recalculate and self.status == self.STATUS_APPROVED_CODE: # FIXME: Should we call this even when status is not approved? - from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job schedule_aggregation_job(self.period) self.period.update_actual_comment() # Update score even when the update is not approved, yet. It handles the @@ -115,7 +115,6 @@ def delete(self, *args, **kwargs): # In case the status was approved, recalculate the period if old_status == self.STATUS_APPROVED_CODE: - from akvo.rsr.usecases.jobs.aggregation import schedule_aggregation_job schedule_aggregation_job(period) self.period.update_actual_comment() self.period.update_score() diff --git a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx index 9d99a903b0..7a18617368 100644 --- a/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/components/ReportedForm.jsx @@ -17,7 +17,6 @@ import { } from '../../../components' import api from '../../../utils/api' import FinalField from '../../../utils/final-field' -import { getMaxDisaggregation, getSumValues, nicenum } from '../../../utils/misc' import RTE from '../../../utils/rte' import ScoringField from '../../../components/ScoringField' import LineChart from '../../../components/LineChart' From 31b231b38ec762530e3f08f58cef1ef11bb2df64 Mon Sep 17 00:00:00 2001 From: zuhdil Date: Tue, 27 Dec 2022 15:54:12 +0700 Subject: [PATCH 59/59] refactor: Cleanup frontend --- akvo/rsr/spa/app/components/PrevUpdate.jsx | 1 - akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/akvo/rsr/spa/app/components/PrevUpdate.jsx b/akvo/rsr/spa/app/components/PrevUpdate.jsx index 56dab42266..58fd2eb624 100644 --- a/akvo/rsr/spa/app/components/PrevUpdate.jsx +++ b/akvo/rsr/spa/app/components/PrevUpdate.jsx @@ -8,7 +8,6 @@ import { groupBy } from 'lodash' import { camelReplace, nicenum } from '../utils/misc' import statusPending from '../images/status-pending.svg' import statusApproved from '../images/status-approved.svg' -import { AllSubmissionsModal } from './AllSubmissionsModal' import { measureType } from '../utils/constants' export const PrevUpdate = ({ diff --git a/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx b/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx index e67f07baf4..e387113933 100644 --- a/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx +++ b/akvo/rsr/spa/app/modules/results-admin/TobeReported.jsx @@ -154,7 +154,7 @@ const TobeReported = ({ grid={{ column: 1 }} itemLayout="vertical" className="tobe-reported" - dataSource={orderBy(updates, ['indicator.title'], ['asc'])} + dataSource={orderBy(dataSource, ['indicator.title'], ['asc'])} renderItem={(item, ix) => { const iKey = item?.id || `${item?.indicator?.id}0${ix}` const updateClass = item?.statusDisplay?.toLowerCase()?.replace(/\s+/g, '-')