Merge pull request #977 from maykinmedia/feature/2040-upgrade-zgw-con…

…sumers

⬆️ [#2040] Upgrade python to 3.11 and zgw-consumers to 0.28.0

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
         uses: maykinmedia/setup-django-backend@v1
         with:
           apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin'
-          python-version: '3.9'
+          python-version: '3.11'
           optimize-postgres: 'yes'
           pg-service: 'postgres'
           setup-node: 'yes'
@@ -119,7 +119,7 @@ jobs:
         uses: maykinmedia/setup-django-backend@v1
         with:
           apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin'
-          python-version: '3.9'
+          python-version: '3.11'
           optimize-postgres: 'yes'
           pg-service: 'postgres'
           setup-node: 'yes'

.github/workflows/code-quality.yml

@@ -25,7 +25,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - uses: isort/isort-action@v0.1.0
         with:
           requirementsFiles: requirements/dev.txt
@@ -46,7 +46,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           pip install -r requirements/dev.txt
@@ -75,7 +75,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install system packages
         run: |
           sudo apt-get update \
@@ -125,7 +125,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install dependencies
         run: |
           pip install -r requirements/dev.txt

.github/workflows/dependencies.yml

@@ -27,7 +27,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Install pip-tools
         run: pip install pip-tools
@@ -49,7 +49,7 @@ jobs:
           submodules: 'true'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
+          python-version: '3.11'
 
       - name: Install pip-tools
         run: pip install pip-tools

.python-version

@@ -1 +1 @@
-3.9
+3.11

Dockerfile

@@ -5,7 +5,7 @@
 
 # Stage 1 - Backend build environment
 # includes compilers and build tooling to create the environment
-FROM python:3.9-slim-buster AS backend-build
+FROM python:3.11-slim-buster AS backend-build
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         pkg-config \
@@ -61,7 +61,7 @@ RUN npm run build
 
 # Stage 3 - Build docker image suitable for production
 
-FROM python:3.9-slim-buster
+FROM python:3.11-slim-buster
 
 # Stage 3.1 - Set up the needed production dependencies
 # Note: mime-support becomes media-types in Debian Bullseye (required for correctly serving mime-types for images)
@@ -93,7 +93,7 @@ RUN mkdir /app/log
 RUN mkdir /app/media
 
 # copy backend build deps
-COPY --from=backend-build /usr/local/lib/python3.9 /usr/local/lib/python3.9
+COPY --from=backend-build /usr/local/lib/python3.11 /usr/local/lib/python3.11
 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
 COPY --from=backend-build /app/src/ /app/src/
 

INSTALL.rst

@@ -19,7 +19,7 @@ Prerequisites
 
 You need the following libraries and/or programs:
 
-* `Python`_ 3.9 or above
+* `Python`_ 3.11 or above
 * Python `Virtualenv`_ and `Pip`_
 * `PostgreSQL`_ 10 or above with PostGIS extension
 * `Node.js`_

README.rst

@@ -6,7 +6,7 @@ Open Inwoner
 :Version: 1.13
 :Source: https://github.com/maykinmedia/open-inwoner
 :Keywords: inwoner
-:PythonVersion: 3.9
+:PythonVersion: 3.11
 
 |build-status| |docker| |black| |python-versions|
 

performance-tests/README.md

@@ -0,0 +1,13 @@
+### Performance tests
+
+To run performance tests:
+1. Ensure that Open Inwoner is running on port 8000
+2. Have Open Inwoner configured to connect with test.openzaak.nl
+3. Run the following commands:
+
+```code
+cd performance-tests
+./tests.sh
+```
+
+The results will be stored in `performance-tests/results/`

performance-tests/locustfile.py

@@ -1,6 +1,7 @@
 from urllib.parse import urlencode
 
 from locust import HttpUser, events, task
+from pyquery import PyQuery as pq
 
 
 class OpenInwonerUser(HttpUser):
@@ -10,6 +11,10 @@ class OpenInwonerUser(HttpUser):
     def mijn_aanvragen_list_cached(self):
         self.client.get("/mijn-aanvragen/content", headers={"HX-Request": "true"})
 
+    @task
+    def mijn_aanvragen_detail_cached(self):
+        self.client.get(f"{self.case_link}content", headers={"HX-Request": "true"})
+
     def on_start(self):
         params = urlencode(
             {
@@ -34,9 +39,18 @@ def on_start(self):
         )
 
         # Ensure uncached call is logged separately
-        self.client.get(
+        response = self.client.get(
             "/mijn-aanvragen/content?_uncached=true", headers={"HX-Request": "true"}
         )
+        doc = pq(response.content)
+
+        self.case_link = pq(
+            doc.find(".cases__link:not([href^='https://www.maykinmedia'])")[0]
+        ).attr("href")
+
+        self.client.get(
+            f"{self.case_link}content?uncached=true", headers={"HX-Request": "true"}
+        )
 
 
 @events.init_command_line_parser.add_listener

performance-tests/results/1.13.2_exceptions.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

performance-tests/results/1.13.2_failures.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

performance-tests/results/1.13.2_stats.csv

@@ -0,0 +1,8 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+GET,/digid/idp/inloggen_ww/?acs=http%3A%2F%2Flocalhost%3A8000%2Fdigid%2Facs%2F&next=http%3A%2F%2Flocalhost%3A8000&cancel=http%3A%2F%2Flocalhost%3A8000%2Faccounts%2Flogin%2F,1,0,146.44589700037614,146.44589700037614,146.44589700037614,146.44589700037614,6738.0,0.01699389075821419,0.0,150,150,150,150,150,150,150,150,150,150,150
+POST,/digid/idp/inloggen_ww/?acs=http%3A%2F%2Flocalhost%3A8000%2Fdigid%2Facs%2F&next=http%3A%2F%2Flocalhost%3A8000&cancel=http%3A%2F%2Flocalhost%3A8000%2Faccounts%2Flogin%2F,1,0,3960.5279700008396,3960.5279700008396,3960.5279700008396,3960.5279700008396,44600.0,0.01699389075821419,0.0,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000
+GET,/mijn-aanvragen/21c55ba6-9ea0-4656-b1e0-8eb51f6cc683/status/content,26,0,500.0,511.9577764229549,474.6624620001967,691.7959559996234,9102.0,0.4418411597135689,0.0,500,500,510,520,530,650,690,690,690,690,690
+GET,/mijn-aanvragen/21c55ba6-9ea0-4656-b1e0-8eb51f6cc683/status/content?uncached=true,1,0,4533.060666999518,4533.060666999518,4533.060666999518,4533.060666999518,9102.0,0.01699389075821419,0.0,4500,4500,4500,4500,4500,4500,4500,4500,4500,4500,4500
+GET,/mijn-aanvragen/content,13,0,670.0,676.1979229231656,622.2663909993571,706.8090819993813,17820.0,0.22092057985678445,0.0,670,690,700,700,710,710,710,710,710,710,710
+GET,/mijn-aanvragen/content?_uncached=true,1,0,9060.142850001284,9060.142850001284,9060.142850001284,9060.142850001284,17895.0,0.01699389075821419,0.0,9100,9100,9100,9100,9100,9100,9100,9100,9100,9100,9100
+,Aggregated,43,0,520.0,925.6198271860465,146.44589700037614,9060.142850001284,12712.720930232557,0.7307373026032101,0.0,520,660,670,690,710,4000,9100,9100,9100,9100,9100

performance-tests/results/1.13.2_stats_history.csv

@@ -0,0 +1,52 @@
+Timestamp,User Count,Type,Name,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%,Total Request Count,Total Failure Count,Total Median Response Time,Total Average Response Time,Total Min Response Time,Total Max Response Time,Total Average Content Size
+1706183628,0,,Aggregated,0.000000,0.000000,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0,0,0,0.0,0,0,0
+1706183629,1,,Aggregated,0.000000,0.000000,150,150,150,150,150,150,150,150,150,150,150,1,0,146.44589700037614,146.44589700037614,146.44589700037614,146.44589700037614,6738.0
+1706183630,1,,Aggregated,0.000000,0.000000,150,150,150,150,150,150,150,150,150,150,150,1,0,146.44589700037614,146.44589700037614,146.44589700037614,146.44589700037614,6738.0
+1706183631,1,,Aggregated,0.000000,0.000000,150,150,150,150,150,150,150,150,150,150,150,1,0,146.44589700037614,146.44589700037614,146.44589700037614,146.44589700037614,6738.0
+1706183632,1,,Aggregated,0.000000,0.000000,150,150,150,150,150,150,150,150,150,150,150,1,0,146.44589700037614,146.44589700037614,146.44589700037614,146.44589700037614,6738.0
+1706183633,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183634,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183635,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183636,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183637,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183638,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183639,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183640,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183641,1,,Aggregated,0.500000,0.000000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,4000,2,0,150.0,2053.486933500608,146.44589700037614,3960.5279700008396,25669.0
+1706183642,1,,Aggregated,0.100000,0.000000,4000,4000,9100,9100,9100,9100,9100,9100,9100,9100,9100,3,0,4000.0,4389.0389056675,146.44589700037614,9060.142850001284,23077.666666666668
+1706183643,1,,Aggregated,0.100000,0.000000,4000,4000,9100,9100,9100,9100,9100,9100,9100,9100,9100,3,0,4000.0,4389.0389056675,146.44589700037614,9060.142850001284,23077.666666666668
+1706183644,1,,Aggregated,0.100000,0.000000,4000,4000,9100,9100,9100,9100,9100,9100,9100,9100,9100,3,0,4000.0,4389.0389056675,146.44589700037614,9060.142850001284,23077.666666666668
+1706183645,1,,Aggregated,0.100000,0.000000,4000,4000,9100,9100,9100,9100,9100,9100,9100,9100,9100,3,0,4000.0,4389.0389056675,146.44589700037614,9060.142850001284,23077.666666666668
+1706183646,1,,Aggregated,0.100000,0.000000,4500,4500,9100,9100,9100,9100,9100,9100,9100,9100,9100,4,0,4000.0,4425.044346000504,146.44589700037614,9060.142850001284,19583.75
+1706183647,1,,Aggregated,0.100000,0.000000,4000,4500,4500,9100,9100,9100,9100,9100,9100,9100,9100,5,0,4000.0,3678.3946680003282,146.44589700037614,9060.142850001284,17487.4
+1706183648,1,,Aggregated,0.100000,0.000000,4000,4000,4500,4500,9100,9100,9100,9100,9100,9100,9100,6,0,690.0,3151.2383440000726,146.44589700037614,9060.142850001284,16089.833333333334
+1706183649,1,,Aggregated,0.200000,0.000000,690,4000,4500,4500,9100,9100,9100,9100,9100,9100,9100,7,0,690.0,2773.0543261428206,146.44589700037614,9060.142850001284,15091.57142857143
+1706183650,1,,Aggregated,0.300000,0.000000,690,4000,4500,4500,9100,9100,9100,9100,9100,9100,9100,8,0,690.0,2512.3069187500278,146.44589700037614,9060.142850001284,15432.625
+1706183651,1,,Aggregated,0.400000,0.000000,690,690,4000,4500,9100,9100,9100,9100,9100,9100,9100,9,0,690.0,2289.109206999961,146.44589700037614,9060.142850001284,14729.222222222223
+1706183652,1,,Aggregated,0.500000,0.000000,690,690,4000,4500,9100,9100,9100,9100,9100,9100,9100,10,0,670.0,2126.8315944998903,146.44589700037614,9060.142850001284,15038.3
+1706183653,1,,Aggregated,0.600000,0.000000,670,690,4000,4000,4500,9100,9100,9100,9100,9100,9100,11,0,670.0,1980.6629140907303,146.44589700037614,9060.142850001284,14498.636363636364
+1706183654,1,,Aggregated,0.600000,0.000000,670,690,4000,4000,4500,9100,9100,9100,9100,9100,9100,12,0,520.0,1856.5188779998607,146.44589700037614,9060.142850001284,14048.916666666666
+1706183655,1,,Aggregated,0.700000,0.000000,520,690,690,4000,4500,9100,9100,9100,9100,9100,9100,13,0,520.0,1751.778898615297,146.44589700037614,9060.142850001284,13668.384615384615
+1706183656,1,,Aggregated,0.800000,0.000000,670,690,690,4000,4500,9100,9100,9100,9100,9100,9100,14,0,520.0,1674.3268104998735,146.44589700037614,9060.142850001284,13964.92857142857
+1706183657,1,,Aggregated,0.800000,0.000000,670,690,690,4000,4500,9100,9100,9100,9100,9100,9100,14,0,520.0,1674.3268104998735,146.44589700037614,9060.142850001284,13964.92857142857
+1706183658,1,,Aggregated,0.900000,0.000000,670,690,700,4000,4500,9100,9100,9100,9100,9100,9100,15,0,670.0,1609.5582465333184,146.44589700037614,9060.142850001284,14221.933333333332
+1706183659,1,,Aggregated,1.000000,0.000000,670,690,700,700,4500,9100,9100,9100,9100,9100,9100,16,0,520.0,1540.1677637499915,146.44589700037614,9060.142850001284,13901.9375
+1706183660,1,,Aggregated,1.000000,0.000000,520,690,690,700,4500,9100,9100,9100,9100,9100,9100,17,0,520.0,1478.9106779999736,146.44589700037614,9060.142850001284,13619.588235294117
+1706183661,1,,Aggregated,1.000000,0.000000,520,670,690,700,4500,9100,9100,9100,9100,9100,9100,18,0,520.0,1424.2436834444259,146.44589700037614,9060.142850001284,13368.611111111111
+1706183662,1,,Aggregated,1.000000,0.000000,520,670,690,700,4500,9100,9100,9100,9100,9100,9100,19,0,520.0,1375.7374102104604,146.44589700037614,9060.142850001284,13144.052631578947
+1706183663,1,,Aggregated,1.000000,0.000000,520,690,700,700,4500,9100,9100,9100,9100,9100,9100,20,0,520.0,1342.0686175999435,146.44589700037614,9060.142850001284,13377.85
+1706183664,1,,Aggregated,1.000000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,21,0,520.0,1302.8671433332333,146.44589700037614,9060.142850001284,13174.238095238095
+1706183665,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,22,0,520.0,1265.7875696817098,146.44589700037614,9060.142850001284,12989.136363636364
+1706183666,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,23,0,520.0,1232.4477851738186,146.44589700037614,9060.142850001284,12820.130434782608
+1706183667,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,24,0,520.0,1207.0235604165493,146.44589700037614,9060.142850001284,13028.458333333334
+1706183668,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,25,0,520.0,1180.022321919896,146.44589700037614,9060.142850001284,12871.4
+1706183669,1,,Aggregated,0.900000,0.000000,520,670,690,690,4000,4500,9100,9100,9100,9100,9100,26,0,520.0,1154.3006668075927,146.44589700037614,9060.142850001284,12726.423076923076
+1706183670,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,27,0,520.0,1137.7233124813856,146.44589700037614,9060.142850001284,12915.074074074075
+1706183671,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,28,0,520.0,1120.7501000356258,146.44589700037614,9060.142850001284,13090.25
+1706183672,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,29,0,520.0,1098.7607715171557,146.44589700037614,9060.142850001284,12952.724137931034
+1706183673,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,29,0,520.0,1098.7607715171557,146.44589700037614,9060.142850001284,12952.724137931034
+1706183674,1,,Aggregated,0.900000,0.000000,520,670,690,700,4000,4500,9100,9100,9100,9100,9100,30,0,520.0,1084.487547066601,146.44589700037614,9060.142850001284,13114.966666666667
+1706183675,1,,Aggregated,0.900000,0.000000,520,670,690,690,710,4500,9100,9100,9100,9100,9100,31,0,520.0,1070.243291677358,146.44589700037614,9060.142850001284,13266.741935483871
+1706183676,1,,Aggregated,1.000000,0.000000,530,670,690,690,710,4500,9100,9100,9100,9100,9100,32,0,520.0,1057.027951093687,146.44589700037614,9060.142850001284,13136.59375
+1706183677,1,,Aggregated,1.000000,0.000000,520,660,670,690,710,4500,9100,9100,9100,9100,9100,33,0,520.0,1039.8739702726414,146.44589700037614,9060.142850001284,13014.333333333334
+1706183678,1,,Aggregated,0.900000,0.000000,530,670,690,700,710,4500,9100,9100,9100,9100,9100,34,0,520.0,1030.0779441469572,146.44589700037614,9060.142850001284,13155.676470588236

performance-tests/results/zgw-consumers-0.28.0_exceptions.csv

@@ -0,0 +1 @@
+Count,Message,Traceback,Nodes

performance-tests/results/zgw-consumers-0.28.0_failures.csv

@@ -0,0 +1 @@
+Method,Name,Error,Occurrences

performance-tests/results/zgw-consumers-0.28.0_stats.csv

@@ -0,0 +1,8 @@
+Type,Name,Request Count,Failure Count,Median Response Time,Average Response Time,Min Response Time,Max Response Time,Average Content Size,Requests/s,Failures/s,50%,66%,75%,80%,90%,95%,98%,99%,99.9%,99.99%,100%
+GET,/digid/idp/inloggen_ww/?acs=http%3A%2F%2Flocalhost%3A8000%2Fdigid%2Facs%2F&next=http%3A%2F%2Flocalhost%3A8000&cancel=http%3A%2F%2Flocalhost%3A8000%2Faccounts%2Flogin%2F,1,0,118.73440300041693,118.73440300041693,118.73440300041693,118.73440300041693,6738.0,0.016998126921107075,0.0,120,120,120,120,120,120,120,120,120,120,120
+POST,/digid/idp/inloggen_ww/?acs=http%3A%2F%2Flocalhost%3A8000%2Fdigid%2Facs%2F&next=http%3A%2F%2Flocalhost%3A8000&cancel=http%3A%2F%2Flocalhost%3A8000%2Faccounts%2Flogin%2F,1,0,1976.276933999543,1976.276933999543,1976.276933999543,1976.276933999543,44600.0,0.016998126921107075,0.0,2000,2000,2000,2000,2000,2000,2000,2000,2000,2000,2000
+GET,/mijn-aanvragen/21c55ba6-9ea0-4656-b1e0-8eb51f6cc683/status/content,21,0,480.0,491.3570622382048,455.9009780004999,555.5192379997607,9102.0,0.3569606653432486,0.0,480,500,510,510,530,550,560,560,560,560,560
+GET,/mijn-aanvragen/21c55ba6-9ea0-4656-b1e0-8eb51f6cc683/status/content?uncached=true,1,0,1517.284825000388,1517.284825000388,1517.284825000388,1517.284825000388,9102.0,0.016998126921107075,0.0,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500,1500
+GET,/mijn-aanvragen/content,24,0,600.0,611.5234143332297,558.8081490004697,678.8694429997122,17820.0,0.40795504610656985,0.0,600,610,640,660,670,670,680,680,680,680,680
+GET,/mijn-aanvragen/content?_uncached=true,1,0,8173.7291119989095,8173.7291119989095,8173.7291119989095,8173.7291119989095,17895.0,0.016998126921107075,0.0,8200,8200,8200,8200,8200,8200,8200,8200,8200,8200,8200
+,Aggregated,49,0,570.0,750.6343984693688,118.73440300041693,8173.7291119989095,14227.69387755102,0.8329082191342467,0.0,570,600,610,630,670,1500,8200,8200,8200,8200,8200