Merge pull request #1015 from maykinmedia/fix/2089-mijn-aanvragen-par…

…allel

⚡ [#2089] Use multithreading for Mijn Aanvragen list

src/open_inwoner/conf/base.py

@@ -896,3 +896,9 @@
 from .app.csp import *  # noqa
 
 SECURE_REFERRER_POLICY = "same-origin"
+
+
+#
+# Project specific settings
+#
+CASE_LIST_NUM_THREADS = 6

src/open_inwoner/openzaak/cases.py

@@ -1,5 +1,10 @@
+import concurrent.futures
 import logging
 
+from django.conf import settings
+
+from zgw_consumers.concurrent import parallel
+
 from .api_models import Zaak
 from .clients import build_client
 from .models import ZaakTypeConfig, ZaakTypeStatusTypeConfig
@@ -82,19 +87,26 @@ def preprocess_data(cases: list[Zaak]) -> list[Zaak]:
     zaken_client = build_client("zaak")
     catalogi_client = build_client("catalogi")
 
+    def preprocess_case(case: Zaak) -> None:
+        resolve_status(case, client=zaken_client)
+        resolve_status_type(case, client=catalogi_client)
+        add_zaak_type_config(case)
+        add_status_type_config(case)
+
     # TODO error handling if these are none?
     # use contextmanager to ensure the `requests.Session` is reused
     with zaken_client, catalogi_client:
-        for case in cases:
-            resolve_zaak_type(case, client=catalogi_client)
+        with parallel(max_workers=settings.CASE_LIST_NUM_THREADS) as executor:
+            futures = [
+                executor.submit(resolve_zaak_type, case, client=catalogi_client)
+                for case in cases
+            ]
+            concurrent.futures.wait(futures)
 
-        cases = [case for case in cases if case.status and is_zaak_visible(case)]
+            cases = [case for case in cases if case.status and is_zaak_visible(case)]
 
-        for case in cases:
-            resolve_status(case, client=zaken_client)
-            resolve_status_type(case, client=catalogi_client)
-            add_zaak_type_config(case)
-            add_status_type_config(case)
+            futures = [executor.submit(preprocess_case, case) for case in cases]
+            concurrent.futures.wait(futures)
 
     cases.sort(key=lambda case: case.startdatum, reverse=True)
 

src/open_inwoner/openzaak/tests/test_cases.py

@@ -3,6 +3,7 @@
 
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
+from django.test import TransactionTestCase
 from django.test.utils import override_settings
 from django.urls import reverse_lazy
 
@@ -139,9 +140,10 @@ def test_no_cases_are_retrieved_when_http_500(self, m):
 
 @requests_mock.Mocker()
 @override_settings(
-    ROOT_URLCONF="open_inwoner.cms.tests.urls", MIDDLEWARE=PATCHED_MIDDLEWARE
+    ROOT_URLCONF="open_inwoner.cms.tests.urls",
+    MIDDLEWARE=PATCHED_MIDDLEWARE,
 )
-class CaseListViewTests(AssertTimelineLogMixin, ClearCachesMixin, WebTest):
+class CaseListViewTests(AssertTimelineLogMixin, ClearCachesMixin, TransactionTestCase):
     inner_url = reverse_lazy("cases:cases_content")
     maxDiff = None
 
@@ -420,9 +422,8 @@ def test_list_cases(self, m):
             call_to_action_text="duplicate",
         )
 
-        response = self.app.get(
-            self.inner_url, user=self.user, headers={"HX-Request": "true"}
-        )
+        self.client.force_login(user=self.user)
+        response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         self.assertListEqual(
             response.context["cases"],
@@ -500,11 +501,8 @@ def test_list_cases_for_eherkenning_user(self, m):
 
                 m.reset_mock()
 
-                response = self.app.get(
-                    self.inner_url,
-                    user=self.eherkenning_user,
-                    headers={"HX-Request": "true"},
-                )
+                self.client.force_login(user=self.eherkenning_user)
+                response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
                 self.assertListEqual(
                     response.context["cases"],
@@ -648,11 +646,8 @@ def test_list_cases_for_eherkenning_user_missing_rsin(self, m):
 
         m.reset_mock()
 
-        response = self.app.get(
-            self.inner_url,
-            user=self.eherkenning_user,
-            headers={"HX-Request": "true"},
-        )
+        self.client.force_login(user=self.eherkenning_user)
+        response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         self.assertListEqual(response.context["cases"], [])
         # don't show internal cases
@@ -670,14 +665,13 @@ def test_list_cases_for_eherkenning_user_missing_rsin(self, m):
     def test_format_zaak_identificatie(self, m):
         config = OpenZaakConfig.get_solo()
         self._setUpMocks(m)
+        self.client.force_login(user=self.user)
 
         with self.subTest("formatting enabled"):
             config.reformat_esuite_zaak_identificatie = True
             config.save()
 
-            response = self.app.get(
-                self.inner_url, user=self.user, headers={"HX-Request": "true"}
-            )
+            response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             e_suite_case = next(
                 (
@@ -693,9 +687,7 @@ def test_format_zaak_identificatie(self, m):
             config.reformat_esuite_zaak_identificatie = False
             config.save()
 
-            response = self.app.get(
-                self.inner_url, user=self.user, headers={"HX-Request": "true"}
-            )
+            response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             e_suite_case = next(
                 (
@@ -727,16 +719,17 @@ def test_list_cases_translates_status(self, m):
             translation="Translated Status Type",
         )
         self._setUpMocks(m)
-        response = self.app.get(
-            self.inner_url, user=self.user, headers={"HX-Request": "true"}
-        )
+        self.client.force_login(user=self.user)
+        response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
+
         self.assertNotContains(response, st1.status)
         self.assertContains(response, st1.translation)
 
     def test_list_cases_logs_displayed_case_ids(self, m):
         self._setUpMocks(m)
 
-        self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+        self.client.force_login(user=self.user)
+        self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         # check access logs for displayed cases
         logs = list(TimelineLog.objects.all())
@@ -769,9 +762,8 @@ def test_list_cases_paginated(self, m):
         self._setUpMocks(m)
 
         # 1. test first page
-        response_1 = self.app.get(
-            self.inner_url, user=self.user, headers={"HX-Request": "true"}
-        )
+        self.client.force_login(user=self.user)
+        response_1 = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         self.assertListEqual(
             response_1.context.get("cases"),
@@ -794,9 +786,7 @@ def test_list_cases_paginated(self, m):
 
         # 2. test next page
         next_page = f"{self.inner_url}?page=2"
-        response_2 = self.app.get(
-            next_page, user=self.user, headers={"HX-Request": "true"}
-        )
+        response_2 = self.client.get(next_page, HTTP_HX_REQUEST="true")
 
         self.assertListEqual(
             response_2.context.get("cases"),
@@ -820,11 +810,10 @@ def test_list_cases_paginated(self, m):
     @patch.object(InnerCaseListView, "paginate_by", 1)
     def test_list_cases_paginated_logs_displayed_case_ids(self, m):
         self._setUpMocks(m)
+        self.client.force_login(user=self.user)
 
         with self.subTest("first page"):
-            response = self.app.get(
-                self.inner_url, user=self.user, headers={"HX-Request": "true"}
-            )
+            response = self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
             self.assertEqual(
                 response.context.get("cases")[0]["uuid"], self.zaak2["uuid"]
             )
@@ -843,9 +832,7 @@ def test_list_cases_paginated_logs_displayed_case_ids(self, m):
 
         with self.subTest("next page"):
             next_page = f"{self.inner_url}?page=2"
-            response = self.app.get(
-                next_page, user=self.user, headers={"HX-Request": "true"}
-            )
+            response = self.client.get(next_page, HTTP_HX_REQUEST="true")
             self.assertEqual(
                 response.context.get("cases")[0]["uuid"], self.zaak1["uuid"]
             )

src/open_inwoner/openzaak/tests/test_cases_cache.py

@@ -1,11 +1,11 @@
 import datetime
 
 from django.core.cache import cache
+from django.test import TransactionTestCase
 from django.test.utils import override_settings
 from django.urls import reverse_lazy
 
 import requests_mock
-from django_webtest import WebTest
 from freezegun import freeze_time
 from furl import furl
 from zgw_consumers.api_models.constants import VertrouwelijkheidsAanduidingen
@@ -23,7 +23,7 @@
 
 @requests_mock.Mocker()
 @override_settings(ROOT_URLCONF="open_inwoner.cms.tests.urls")
-class OpenCaseListCacheTests(ClearCachesMixin, WebTest):
+class OpenCaseListCacheTests(ClearCachesMixin, TransactionTestCase):
     inner_url = reverse_lazy("cases:cases_content")
 
     def setUp(self):
@@ -198,7 +198,8 @@ def test_case_types_are_cached(self, m):
         # Cache is empty before the request
         self.assertIsNone(cache.get(f"case_type:{self.zaaktype['url']}"))
 
-        self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+        self.client.force_login(user=self.user)
+        self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         # Case type is cached after the request
         self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}"))
@@ -207,7 +208,8 @@ def test_cached_case_types_are_deleted_after_one_day(self, m):
         self._setUpMocks(m)
 
         with freeze_time("2022-01-01 12:00") as frozen_time:
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.force_login(user=self.user)
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             # After one day the results should be deleted
             frozen_time.tick(delta=datetime.timedelta(days=1))
@@ -220,7 +222,8 @@ def test_cached_case_types_in_combination_with_new_ones(self, m):
             # First attempt
             self.assertIsNone(cache.get(f"case_type:{self.zaaktype['url']}"))
 
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.force_login(user=self.user)
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}"))
 
@@ -230,7 +233,7 @@ def test_cached_case_types_in_combination_with_new_ones(self, m):
             frozen_time.tick(delta=datetime.timedelta(minutes=3))
             self.assertIsNone(cache.get(f"case_type:{self.new_zaaktype['url']}"))
 
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             self.assertIsNotNone(cache.get(f"case_type:{self.zaaktype['url']}"))
             self.assertIsNotNone(cache.get(f"case_type:{self.new_zaaktype['url']}"))
@@ -239,7 +242,8 @@ def test_cached_status_types_are_deleted_after_one_day(self, m):
         self._setUpMocks(m)
 
         with freeze_time("2022-01-01 12:00") as frozen_time:
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.force_login(user=self.user)
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             # After one day the results should be deleted
             frozen_time.tick(delta=datetime.timedelta(hours=24))
@@ -257,7 +261,8 @@ def test_statuses_are_cached(self, m):
         self.assertIsNone(cache.get(f"status:{self.status1['url']}"))
         self.assertIsNone(cache.get(f"status:{self.status2['url']}"))
 
-        self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+        self.client.force_login(user=self.user)
+        self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
         # Status is cached after the request
         self.assertIsNotNone(cache.get(f"status:{self.status1['url']}"))
@@ -267,7 +272,8 @@ def test_cached_statuses_are_deleted_after_one_hour(self, m):
         self._setUpMocks(m)
 
         with freeze_time("2022-01-01 12:00") as frozen_time:
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.force_login(user=self.user)
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             # After one hour the results should be deleted
             frozen_time.tick(delta=datetime.timedelta(hours=1))
@@ -282,7 +288,8 @@ def test_cached_statuses_in_combination_with_new_ones(self, m):
             self.assertIsNone(cache.get(f"status:{self.status1['url']}"))
             self.assertIsNone(cache.get(f"status:{self.status2['url']}"))
 
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.force_login(user=self.user)
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             self.assertIsNotNone(cache.get(f"status:{self.status1['url']}"))
             self.assertIsNotNone(cache.get(f"status:{self.status2['url']}"))
@@ -293,7 +300,7 @@ def test_cached_statuses_in_combination_with_new_ones(self, m):
             frozen_time.tick(delta=datetime.timedelta(minutes=3))
             self.assertIsNone(cache.get(f"status:{self.new_status['url']}"))
 
-            self.app.get(self.inner_url, user=self.user, headers={"HX-Request": "true"})
+            self.client.get(self.inner_url, HTTP_HX_REQUEST="true")
 
             self.assertIsNotNone(cache.get(f"status:{self.new_status['url']}"))
             self.assertIsNotNone(cache.get(f"status:{self.status1['url']}"))