Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[#2764] Resolve cases concurrently when loading the case list #1407

Merged
merged 7 commits into from
Oct 2, 2024
Merged

[#2764] Resolve cases concurrently when loading the case list #1407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cade125
Properly type client getters on ZGWApiGroupConfig
swrichards Sep 19, 2024
260cf75
Properly implement UniformCase protocol
swrichards Sep 19, 2024
dcfbcf7
Directly resolve zaak status in notification handler
swrichards Sep 19, 2024
9c76b5b
[#2764] Resolve cases in case list concurrently
swrichards Sep 19, 2024
d74e8aa
Refactor CaseListService into its own submodule
swrichards Oct 1, 2024
3190ba6
Improve typing and invariant checks for CaseListService
swrichards Oct 1, 2024
5cf9e4c
Add a clarifying comment wrt thread limits in CaseListService
swrichards Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 5 additions & 138 deletions src/open_inwoner/cms/cases/views/cases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,159 +1,26 @@
import concurrent.futures
import enum
import logging
from dataclasses import dataclass
from typing import Sequence

from django.http import HttpRequest
from django.urls import reverse
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from django.views.generic import TemplateView

from furl import furl
from view_breadcrumbs import BaseBreadcrumbMixin
from zgw_consumers.concurrent import parallel

from open_inwoner.htmx.mixins import RequiresHtmxMixin
from open_inwoner.openzaak.api_models import OpenSubmission, Zaak
from open_inwoner.openzaak.cases import preprocess_data
from open_inwoner.openzaak.models import OpenZaakConfig, ZGWApiGroupConfig
from open_inwoner.openzaak.models import OpenZaakConfig
from open_inwoner.openzaak.types import UniformCase
from open_inwoner.openzaak.utils import get_user_fetch_parameters
from open_inwoner.utils.mixins import PaginationMixin
from open_inwoner.utils.views import CommonPageMixin

from .mixins import CaseAccessMixin, CaseLogMixin, OuterCaseAccessMixin
from .services import CaseFilterFormOption, CaseListService

logger = logging.getLogger(__name__)


class CaseFilterFormOption(enum.Enum):
OPEN_SUBMISSION = _("Openstaande formulieren")
OPEN_CASE = _("Lopende aanvragen")
CLOSED_CASE = _("Afgeronde aanvragen")


@dataclass(frozen=True)
class ZaakWithApiGroup(UniformCase):
zaak: Zaak
api_group: ZGWApiGroupConfig

@property
def identifier(self):
return self.zaak.url

def process_data(self) -> dict:
return {**self.zaak.process_data(), "api_group": self.api_group}


@dataclass(frozen=True)
class SubmissionWithApiGroup:
submission: OpenSubmission
api_group: ZGWApiGroupConfig

@property
def identifier(self):
return self.submission.url

def process_data(self) -> dict:
return {**self.submission.process_data(), "api_group": self.api_group}


class CaseListService:
def __init__(self, request: HttpRequest):
self.request = request

def get_cases_for_api_group(self, group: ZGWApiGroupConfig) -> list[UniformCase]:
raw_cases = group.zaken_client.fetch_cases(
**get_user_fetch_parameters(self.request, check_rsin=True)
)
preprocessed_cases = preprocess_data(raw_cases, group)
return preprocessed_cases

def get_submissions_for_api_group(
self, group: ZGWApiGroupConfig
) -> list[UniformCase]:
return group.forms_client.fetch_open_submissions(
**get_user_fetch_parameters(self.request, check_rsin=False)
)

def get_cases(self) -> list[ZaakWithApiGroup]:
all_api_groups = list(ZGWApiGroupConfig.objects.all())

with parallel() as executor:
futures = [
executor.submit(self.get_cases_for_api_group, group)
for group in all_api_groups
]

cases_with_api_group = []
for task in concurrent.futures.as_completed(futures):
try:
group_for_task = all_api_groups[futures.index(task)]
for row in task.result():
cases_with_api_group.append(
ZaakWithApiGroup(zaak=row, api_group=group_for_task)
)
except BaseException:
logger.exception("Error fetching and pre-processing cases")

# Ensure stable sorting for pagination and testing purposes
cases_with_api_group.sort(key=lambda c: all_api_groups.index(c.api_group))

return cases_with_api_group

def get_submissions(self):
all_api_groups = list(
ZGWApiGroupConfig.objects.exclude(form_service__isnull=True)
)

with parallel() as executor:
futures = [
executor.submit(self.get_submissions_for_api_group, group)
for group in all_api_groups
]

subs_with_api_group = []
for task in concurrent.futures.as_completed(futures):
try:
group_for_task = all_api_groups[futures.index(task)]
for row in task.result():
subs_with_api_group.append(
SubmissionWithApiGroup(
submission=row, api_group=group_for_task
)
)
except BaseException:
logger.exception("Error fetching and pre-processing cases")

# Sort submissions by date modified
subs_with_api_group.sort(
key=lambda sub: sub.submission.datum_laatste_wijziging, reverse=True
)

return subs_with_api_group

@staticmethod
def get_case_filter_status(zaak: Zaak) -> CaseFilterFormOption:
if zaak.einddatum:
return CaseFilterFormOption.CLOSED_CASE

return CaseFilterFormOption.OPEN_CASE

def get_case_status_frequencies(self) -> dict[CaseFilterFormOption, int]:
cases = self.get_cases()
submissions = self.get_submissions()

case_statuses = [self.get_case_filter_status(case.zaak) for case in cases]

# add static text for open submissions
case_statuses += [CaseFilterFormOption.OPEN_SUBMISSION for _ in submissions]

return {
status: case_statuses.count(status) for status in list(CaseFilterFormOption)
}


class OuterCaseListView(
OuterCaseAccessMixin, CommonPageMixin, BaseBreadcrumbMixin, TemplateView
):
Expand Down Expand Up @@ -203,8 +70,8 @@ def get_context_data(self, **kwargs):
context["filter_form_enabled"] = config.zaken_filter_enabled

# update ctx with open submissions and cases (possibly fitered)
open_submissions: list[UniformCase] = case_service.get_submissions()
preprocessed_cases: list[UniformCase] = case_service.get_cases()
open_submissions: Sequence[UniformCase] = case_service.get_submissions()
preprocessed_cases: Sequence[UniformCase] = case_service.get_cases()

if config.zaken_filter_enabled:
case_status_frequencies = case_service.get_case_status_frequencies()
Expand Down
Loading
Loading