From 40d48dd4323b9539f2c9099158f718540fa6f1d8 Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 9 Feb 2024 14:20:07 +0100 Subject: [PATCH] :zap: [#2089] Use multithreading for Mijn Aanvragen list issue: https://taiga.maykinmedia.nl/project/open-inwoner/issue/2089 --- src/open_inwoner/conf/base.py | 6 ++ src/open_inwoner/openzaak/cases.py | 28 ++++++--- src/open_inwoner/openzaak/tests/test_cases.py | 61 ++++++++----------- .../openzaak/tests/test_cases_cache.py | 29 +++++---- 4 files changed, 68 insertions(+), 56 deletions(-) diff --git a/src/open_inwoner/conf/base.py b/src/open_inwoner/conf/base.py index d5a79594ca..4170a9a88a 100644 --- a/src/open_inwoner/conf/base.py +++ b/src/open_inwoner/conf/base.py @@ -896,3 +896,9 @@ from .app.csp import * # noqa SECURE_REFERRER_POLICY = "same-origin" + + +# +# Project specific settings +# +CASE_LIST_NUM_THREADS = 6 diff --git a/src/open_inwoner/openzaak/cases.py b/src/open_inwoner/openzaak/cases.py index 75a5c6f064..697c2f0845 100644 --- a/src/open_inwoner/openzaak/cases.py +++ b/src/open_inwoner/openzaak/cases.py @@ -1,5 +1,10 @@ +import concurrent.futures import logging +from django.conf import settings + +from zgw_consumers.concurrent import parallel + from .api_models import Zaak from .clients import build_client from .models import ZaakTypeConfig, ZaakTypeStatusTypeConfig @@ -82,19 +87,26 @@ def preprocess_data(cases: list[Zaak]) -> list[Zaak]: zaken_client = build_client("zaak") catalogi_client = build_client("catalogi") + def preprocess_case(case: Zaak) -> None: + resolve_status(case, client=zaken_client) + resolve_status_type(case, client=catalogi_client) + add_zaak_type_config(case) + add_status_type_config(case) + # TODO error handling if these are none? # use contextmanager to ensure the `requests.Session` is reused with zaken_client, catalogi_client: - for case in cases: - resolve_zaak_type(case, client=catalogi_client) + with parallel(max_workers=settings.CASE_LIST_NUM_THREADS) as executor: + futures = [ + executor.submit(resolve_zaak_type, case, client=catalogi_client) + for case in cases + ] + concurrent.futures.wait(futures) - cases = [case for case in cases if case.status and is_zaak_visible(case)] + cases = [case for case in cases if case.status and is_zaak_visible(case)] - for case in cases: - resolve_status(case, client=zaken_client) - resolve_status_type(case, client=catalogi_client) - add_zaak_type_config(case) - add_status_type_config(case) + futures = [executor.submit(preprocess_case, case) for case in cases] + concurrent.futures.wait(futures) cases.sort(key=lambda case: case.startdatum, reverse=True) diff --git a/src/open_inwoner/openzaak/tests/test_cases.py b/src/open_inwoner/openzaak/tests/test_cases.py index b90464c912..4179e3913a 100644 --- a/src/open_inwoner/openzaak/tests/test_cases.py +++ b/src/open_inwoner/openzaak/tests/test_cases.py @@ -3,6 +3,7 @@ from django.conf import settings from django.contrib.auth.models import AnonymousUser +from django.test import TransactionTestCase from django.test.utils import override_settings from django.urls import reverse_lazy @@ -139,9 +140,10 @@ def test_no_cases_are_retrieved_when_http_500(self, m): @requests_mock.Mocker() @override_settings( - ROOT_URLCONF="open_inwoner.cms.tests.urls", MIDDLEWARE=PATCHED_MIDDLEWARE + ROOT_URLCONF="open_inwoner.cms.tests.urls", + MIDDLEWARE=PATCHED_MIDDLEWARE, ) -class CaseListViewTests(AssertTimelineLogMixin, ClearCachesMixin, WebTest): +class CaseListViewTests(AssertTimelineLogMixin, ClearCachesMixin, TransactionTestCase): inner_url = reverse_lazy("cases:cases_content") maxDiff = None @@ -420,9 +422,8 @@ def test_list_cases(self, m): call_to_action_text="duplicate", ) - response = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + self.client.force_login(user=self.user) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertListEqual( response.context["cases"], @@ -500,11 +501,8 @@ def test_list_cases_for_eherkenning_user(self, m): m.reset_mock() - response = self.app.get( - self.inner_url, - user=self.eherkenning_user, - headers={"HX-Request": "true"}, - ) + self.client.force_login(user=self.eherkenning_user) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertListEqual( response.context["cases"], @@ -648,11 +646,8 @@ def test_list_cases_for_eherkenning_user_missing_rsin(self, m): m.reset_mock() - response = self.app.get( - self.inner_url, - user=self.eherkenning_user, - headers={"HX-Request": "true"}, - ) + self.client.force_login(user=self.eherkenning_user) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertListEqual(response.context["cases"], []) # don't show internal cases @@ -670,14 +665,13 @@ def test_list_cases_for_eherkenning_user_missing_rsin(self, m): def test_format_zaak_identificatie(self, m): config = OpenZaakConfig.get_solo() self._setUpMocks(m) + self.client.force_login(user=self.user) with self.subTest("formatting enabled"): config.reformat_esuite_zaak_identificatie = True config.save() - response = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") e_suite_case = next( ( @@ -693,9 +687,7 @@ def test_format_zaak_identificatie(self, m): config.reformat_esuite_zaak_identificatie = False config.save() - response = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") e_suite_case = next( ( @@ -727,16 +719,17 @@ def test_list_cases_translates_status(self, m): translation="Translated Status Type", ) self._setUpMocks(m) - response = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + self.client.force_login(user=self.user) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") + self.assertNotContains(response, st1.status) self.assertContains(response, st1.translation) def test_list_cases_logs_displayed_case_ids(self, m): self._setUpMocks(m) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # check access logs for displayed cases logs = list(TimelineLog.objects.all()) @@ -769,9 +762,8 @@ def test_list_cases_paginated(self, m): self._setUpMocks(m) # 1. test first page - response_1 = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + self.client.force_login(user=self.user) + response_1 = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertListEqual( response_1.context.get("cases"), @@ -794,9 +786,7 @@ def test_list_cases_paginated(self, m): # 2. test next page next_page = f"{self.inner_url}?page=2" - response_2 = self.app.get( - next_page, user=self.user, headers={"HX-Request": "true"} - ) + response_2 = self.client.get(next_page, HTTP_HX_REQUEST="true") self.assertListEqual( response_2.context.get("cases"), @@ -820,11 +810,10 @@ def test_list_cases_paginated(self, m): @patch.object(InnerCaseListView, "paginate_by", 1) def test_list_cases_paginated_logs_displayed_case_ids(self, m): self._setUpMocks(m) + self.client.force_login(user=self.user) with self.subTest("first page"): - response = self.app.get( - self.inner_url, user=self.user, headers={"HX-Request": "true"} - ) + response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertEqual( response.context.get("cases")[0]["uuid"], self.zaak2["uuid"] ) @@ -843,9 +832,7 @@ def test_list_cases_paginated_logs_displayed_case_ids(self, m): with self.subTest("next page"): next_page = f"{self.inner_url}?page=2" - response = self.app.get( - next_page, user=self.user, headers={"HX-Request": "true"} - ) + response = self.client.get(next_page, HTTP_HX_REQUEST="true") self.assertEqual( response.context.get("cases")[0]["uuid"], self.zaak1["uuid"] ) diff --git a/src/open_inwoner/openzaak/tests/test_cases_cache.py b/src/open_inwoner/openzaak/tests/test_cases_cache.py index 0791d3747c..24cc007b27 100644 --- a/src/open_inwoner/openzaak/tests/test_cases_cache.py +++ b/src/open_inwoner/openzaak/tests/test_cases_cache.py @@ -1,11 +1,11 @@ import datetime from django.core.cache import cache +from django.test import TransactionTestCase from django.test.utils import override_settings from django.urls import reverse_lazy import requests_mock -from django_webtest import WebTest from freezegun import freeze_time from furl import furl from zgw_consumers.api_models.constants import VertrouwelijkheidsAanduidingen @@ -23,7 +23,7 @@ @requests_mock.Mocker() @override_settings(ROOT_URLCONF="open_inwoner.cms.tests.urls") -class OpenCaseListCacheTests(ClearCachesMixin, WebTest): +class OpenCaseListCacheTests(ClearCachesMixin, TransactionTestCase): inner_url = reverse_lazy("cases:cases_content") def setUp(self): @@ -198,7 +198,8 @@ def test_case_types_are_cached(self, m): # Cache is empty before the request self.assertIsNone(cache.get(f"case_type:{self.zaaktype['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # Case type is cached after the request self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}")) @@ -207,7 +208,8 @@ def test_cached_case_types_are_deleted_after_one_day(self, m): self._setUpMocks(m) with freeze_time("2022-01-01 12:00") as frozen_time: - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # After one day the results should be deleted frozen_time.tick(delta=datetime.timedelta(days=1)) @@ -220,7 +222,8 @@ def test_cached_case_types_in_combination_with_new_ones(self, m): # First attempt self.assertIsNone(cache.get(f"case_type:{self.zaaktype['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}")) @@ -230,7 +233,7 @@ def test_cached_case_types_in_combination_with_new_ones(self, m): frozen_time.tick(delta=datetime.timedelta(minutes=3)) self.assertIsNone(cache.get(f"case_type:{self.new_zaaktype['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}")) self.assertIsNotNone(cache.get(f"case_type:{self.new_zaaktype['url']}")) @@ -239,7 +242,8 @@ def test_cached_status_types_are_deleted_after_one_day(self, m): self._setUpMocks(m) with freeze_time("2022-01-01 12:00") as frozen_time: - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # After one day the results should be deleted frozen_time.tick(delta=datetime.timedelta(hours=24)) @@ -257,7 +261,8 @@ def test_statuses_are_cached(self, m): self.assertIsNone(cache.get(f"status:{self.status1['url']}")) self.assertIsNone(cache.get(f"status:{self.status2['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # Status is cached after the request self.assertIsNotNone(cache.get(f"status:{self.status1['url']}")) @@ -267,7 +272,8 @@ def test_cached_statuses_are_deleted_after_one_hour(self, m): self._setUpMocks(m) with freeze_time("2022-01-01 12:00") as frozen_time: - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") # After one hour the results should be deleted frozen_time.tick(delta=datetime.timedelta(hours=1)) @@ -282,7 +288,8 @@ def test_cached_statuses_in_combination_with_new_ones(self, m): self.assertIsNone(cache.get(f"status:{self.status1['url']}")) self.assertIsNone(cache.get(f"status:{self.status2['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.force_login(user=self.user) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertIsNotNone(cache.get(f"status:{self.status1['url']}")) self.assertIsNotNone(cache.get(f"status:{self.status2['url']}")) @@ -293,7 +300,7 @@ def test_cached_statuses_in_combination_with_new_ones(self, m): frozen_time.tick(delta=datetime.timedelta(minutes=3)) self.assertIsNone(cache.get(f"status:{self.new_status['url']}")) - self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"}) + self.client.get(self.inner_url, HTTP_HX_REQUEST="true") self.assertIsNotNone(cache.get(f"status:{self.new_status['url']}")) self.assertIsNotNone(cache.get(f"status:{self.status1['url']}"))