Merge pull request #908 from maykinmedia/issue/1865-mijn-aanvragen-pe…

…rformance

⚡ [#1865/2017] Add basic locust setup & increase catalogi API request timeout default

performance-tests/locustfile.py

@@ -0,0 +1,49 @@
+from urllib.parse import urlencode
+
+from locust import HttpUser, events, task
+
+
+class OpenInwonerUser(HttpUser):
+    wait_time = lambda _: 0.5
+
+    @task
+    def mijn_aanvragen_list_cached(self):
+        self.client.get("/mijn-aanvragen/content", headers={"HX-Request": "true"})
+
+    def on_start(self):
+        params = urlencode(
+            {
+                "acs": f"{self.host}/digid/acs/",
+                "next": self.host,
+                "cancel": f"{self.host}/accounts/login/",
+            }
+        )
+        digid_login_url = f"/digid/idp/inloggen_ww/?{params}"
+
+        response = self.client.get(digid_login_url)
+        csrftoken = response.cookies["csrftoken"]
+
+        self.client.post(
+            digid_login_url,
+            data={
+                "auth_name": self.bsn,
+                "auth_pass": "foo",
+                "csrfmiddlewaretoken": csrftoken,
+                "commit": "Inloggen",
+            },
+        )
+
+        # Ensure uncached call is logged separately
+        self.client.get(
+            "/mijn-aanvragen/content?_uncached=true", headers={"HX-Request": "true"}
+        )
+
+
+@events.init_command_line_parser.add_listener
+def init_parser(parser):
+    parser.add_argument("--bsn", type=str, dest="bsn", help="BSN used to login with")
+
+
+@events.init.add_listener
+def on_locust_init(environment, **kwargs):
+    OpenInwonerUser.bsn = environment.parsed_options.bsn

requirements/dev.in

@@ -17,3 +17,6 @@ django-extensions
 # Documentation
 sphinx
 sphinx_rtd_theme
+
+# performance testing
+locust

requirements/dev.txt

@@ -47,6 +47,8 @@ billiard==3.6.4.0
     #   celery
 black==22.12.0
     # via -r requirements/dev.in
+blinker==1.7.0
+    # via flask
 boltons==21.0.0
     # via
     #   -c requirements/ci.txt
@@ -58,6 +60,7 @@ brotli==1.0.9
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   fonttools
+    #   geventhttpclient
 build==0.8.0
     # via pip-tools
 celery==5.2.7
@@ -72,6 +75,7 @@ certifi==2023.11.17
     #   django-simple-certmanager
     #   elastic-apm
     #   elasticsearch
+    #   geventhttpclient
     #   requests
     #   sentry-sdk
 cffi==1.15.0
@@ -94,6 +98,7 @@ click==8.1.3
     #   click-didyoumean
     #   click-plugins
     #   click-repl
+    #   flask
     #   pip-tools
 click-didyoumean==0.3.0
     # via
@@ -110,6 +115,8 @@ click-repl==0.2.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   celery
+configargparse==1.7
+    # via locust
 confusable-homoglyphs==3.2.0
     # via
     #   -c requirements/ci.txt
@@ -534,6 +541,15 @@ faker==9.9.0
     #   zgw-consumers
 flake8==3.9.2
     # via -r requirements/dev.in
+flask==3.0.0
+    # via
+    #   flask-basicauth
+    #   flask-cors
+    #   locust
+flask-basicauth==0.2.0
+    # via locust
+flask-cors==4.0.0
+    # via locust
 fontawesomefree==6.4.2
     # via
     #   -c requirements/ci.txt
@@ -567,6 +583,12 @@ geopy==2.2.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+gevent==23.9.1
+    # via
+    #   geventhttpclient
+    #   locust
+geventhttpclient==2.0.11
+    # via locust
 gitdb==4.0.9
     # via gitpython
 gitpython==3.1.27
@@ -580,6 +602,7 @@ greenlet==2.0.2
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   gevent
     #   playwright
 html5lib==1.1
     # via
@@ -602,6 +625,7 @@ importlib-metadata==4.8.2
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+    #   flask
     #   markdown
 inflection==0.5.1
     # via
@@ -619,8 +643,12 @@ isort==5.9.3
     #   -r requirements/ci.txt
     #   -r requirements/dev.in
     #   pylint
-jinja2==3.0.1
-    # via sphinx
+itsdangerous==2.1.2
+    # via flask
+jinja2==3.1.2
+    # via
+    #   flask
+    #   sphinx
 josepy==1.13.0
     # via
     #   -c requirements/ci.txt
@@ -646,6 +674,8 @@ lockfile==0.12.2
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   django-yubin
+locust==2.20.0
+    # via -r requirements/dev.in
 lxml==4.9.1
     # via
     #   -c requirements/ci.txt
@@ -673,8 +703,10 @@ markuppy==1.14
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   tablib
-markupsafe==2.0.1
-    # via jinja2
+markupsafe==2.1.3
+    # via
+    #   jinja2
+    #   werkzeug
 maykin-django-two-factor-auth==2.0.4
     # via
     #   -c requirements/ci.txt
@@ -703,6 +735,8 @@ mozilla-django-oidc-db==0.12.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
+msgpack==1.0.7
+    # via locust
 mypy-extensions==0.4.3
     # via black
 notifications-api-common==0.2.0
@@ -779,6 +813,8 @@ prompt-toolkit==3.0.36
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   click-repl
+psutil==5.9.7
+    # via locust
 psycopg2==2.9.1
     # via
     #   -c requirements/ci.txt
@@ -878,6 +914,8 @@ pyyaml==6.0
     #   drf-spectacular
     #   gemma-zds-client
     #   tablib
+pyzmq==25.1.2
+    # via locust
 qrcode==6.1
     # via
     #   -c requirements/ci.txt
@@ -903,6 +941,7 @@ requests==2.31.0
     #   django-open-forms-client
     #   django-rosetta
     #   gemma-zds-client
+    #   locust
     #   maykin-python3-saml
     #   messagebird
     #   mozilla-django-oidc
@@ -915,6 +954,8 @@ requests-mock==1.9.3
     #   -r requirements/ci.txt
     #   ddt-api-calls
     #   zgw-consumers
+roundrobin==0.0.4
+    # via locust
 sentry-sdk==1.38.0
     # via
     #   -c requirements/ci.txt
@@ -933,6 +974,7 @@ six==1.16.0
     #   django-elasticsearch-dsl
     #   elasticsearch-dsl
     #   furl
+    #   geventhttpclient
     #   html5lib
     #   isodate
     #   mail-parser
@@ -1075,6 +1117,10 @@ webtest==3.0.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   django-webtest
+werkzeug==3.0.1
+    # via
+    #   flask
+    #   locust
 wheel==0.42.0
     # via pip-tools
 wrapt==1.12.1
@@ -1111,6 +1157,10 @@ zipp==3.6.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   importlib-metadata
+zope-event==5.0
+    # via gevent
+zope-interface==6.1
+    # via gevent
 zopfli==0.1.9
     # via
     #   -c requirements/ci.txt

src/open_inwoner/conf/base.py

@@ -110,7 +110,7 @@
 SOLO_CACHE = "local"  # Avoid Redis overhead
 
 # ZGW API caches
-CACHE_ZGW_CATALOGI_TIMEOUT = config("CACHE_ZGW_CATALOGI_TIMEOUT", default=60 * 15)
+CACHE_ZGW_CATALOGI_TIMEOUT = config("CACHE_ZGW_CATALOGI_TIMEOUT", default=60 * 60 * 24)
 CACHE_ZGW_ZAKEN_TIMEOUT = config("CACHE_ZGW_ZAKEN_TIMEOUT", default=60 * 1)
 
 #