Merge pull request #1314 from maykinmedia/feature/docker-love

Docker / Docker-compose improvements

.nvmrc

@@ -1 +1 @@
-12
+20

Dockerfile

@@ -7,7 +7,7 @@
 # includes compilers and build tooling to create the environment
 FROM python:3.11-slim-bookworm AS backend-build
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
         pkg-config \
         build-essential \
         git \
@@ -24,21 +24,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
-RUN mkdir /app/src
 
-# Ensure we use the latest version of pip
-RUN pip install pip>=24 setuptools -U
+# Use uv to install dependencies
+RUN pip install uv -U
 COPY ./requirements /app/requirements
-RUN pip install -r requirements/production.txt
 
-
-# Stage 2 - Install frontend deps and build assets
-FROM node:20-buster AS frontend-build
+RUN uv pip install --system -r requirements/production.txt
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
     && rm -rf /var/lib/apt/lists/*
 
+# Stage 2 - Install frontend deps and build assets
+FROM node:20-bookworm-slim AS frontend-build
+
 WORKDIR /app
 
 # copy configuration/build files
@@ -54,19 +53,17 @@ COPY ./src /app/src
 # build frontend
 RUN npm run build
 
-
 # Stage 3 - Build docker image suitable for production
-
 FROM python:3.11-slim-bookworm
 
 # Stage 3.1 - Set up the needed production dependencies
 # Note: mime-support becomes media-types in Debian Bullseye (required for correctly serving mime-types for images)
 # Also install the dependencies for GeoDjango
-
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
         curl \
         procps \
-        vim \
+        nano \
+        mime-support \
         postgresql-client \
         libgdal32 \
         libgeos-c1v5 \
@@ -75,8 +72,8 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
         libxmlsec1-openssl \
         libgdk-pixbuf2.0-0 \
         libffi-dev \
+        gettext \
         shared-mime-info \
-        mime-support \
         # weasyprint deps (https://doc.courtbouillon.org/weasyprint/stable/first_steps.html#debian-11)
         libpango-1.0-0 \
         libpangoft2-1.0-0 \
@@ -86,15 +83,19 @@ WORKDIR /app
 COPY ./bin/docker_start.sh /start.sh
 COPY ./bin/wait_for_db.sh /wait_for_db.sh
 COPY ./bin/celery_worker.sh /celery_worker.sh
-COPY ./bin/check_celery_worker_liveness.py ./bin/
+COPY ./bin/celery_beat.sh /celery_beat.sh
+COPY ./bin/celery_monitor.sh /celery_monitor.sh
 COPY ./bin/setup_configuration.sh /setup_configuration.sh
-RUN mkdir /app/log
-RUN mkdir /app/media
+RUN mkdir /app/log /app/media /app/private_media /app/tmp
+COPY ./bin/check_celery_worker_liveness.py ./bin/
+
+# prevent writing to the container layer, which would degrade performance.
+# This also serves as a hint for the intended volumes.
+VOLUME ["/app/log", "/app/media", "/app/private_media"]
 
 # copy backend build deps
 COPY --from=backend-build /usr/local/lib/python3.11 /usr/local/lib/python3.11
 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
-COPY --from=backend-build /app/src/ /app/src/
 COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
 
 # copy frontend build statics
@@ -109,16 +110,26 @@ RUN chown -R maykin /app
 # drop privileges
 USER maykin
 
-ARG COMMIT_HASH
+ARG RELEASE COMMIT_HASH
 ENV GIT_SHA=${COMMIT_HASH}
+ENV RELEASE=${RELEASE}
+
 ENV DJANGO_SETTINGS_MODULE=open_inwoner.conf.docker
+
 ENV DIGID_MOCK=True
 ENV EHERKENNING_MOCK=True
 
 ARG SECRET_KEY=dummy
 
-# Run collectstatic, so the result is already included in the image
-RUN python src/manage.py collectstatic --noinput
+LABEL org.label-schema.vcs-ref=$COMMIT_HASH \
+      org.label-schema.vcs-url="https://github.com/maykinmedia/open-inwoner" \
+      org.label-schema.version=$RELEASE \
+      org.label-schema.name="Open Inwoner"
+
+# Run collectstatic and compilemessages, so the result is already included in
+# the image
+RUN python src/manage.py collectstatic --noinput \
+    && python src/manage.py compilemessages
 
 EXPOSE 8000
 CMD ["/start.sh"]

bin/celery_beat.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+
+mkdir -p celerybeat
+
+# Adding the database scheduler will also convert the CELERY_BEAT_SCHEDULE to
+# database entries.
+
+echo "Starting celery beat"
+exec celery --workdir src --app "open_inwoner.celery" beat \
+    -l $LOGLEVEL \
+    -s ../celerybeat/beat \
+    --scheduler django_celery_beat.schedulers:DatabaseScheduler \
+    --pidfile=  # empty on purpose, see https://github.com/open-formulieren/open-forms/issues/1182

bin/celery_monitor.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+
+# This monitors the tasks executed by Celery. The Celery worker needs to be
+# started with the -E option to sent out the events.
+
+echo "Starting celery events"
+exec celery --workdir src --app "open_inwoner.celery" events \
+    -l $LOGLEVEL \
+    --camera django_celery_monitor.camera.Camera \
+    --frequency=2.0

bin/celery_worker.sh

@@ -15,5 +15,11 @@ if [[ "$ENABLE_COVERAGE" ]]; then
 fi
 
 echo "Starting celery worker $WORKER_NAME with queue $QUEUE"
-exec $_binary --workdir src -A "open_inwoner" events -l info --camera django_celery_monitor.camera.Camera --frequency=2.0 &
-exec $_binary --workdir src -A "open_inwoner" worker -l $LOGLEVEL -c $CONCURRENCY -Q $QUEUE -n $WORKER_NAME -E --max-tasks-per-child=50 -l info -B --scheduler django_celery_beat.schedulers:DatabaseScheduler
+exec $_binary --workdir src --app "open_inwoner.celery" worker \
+    -Q $QUEUE \
+    -n $WORKER_NAME \
+    -l $LOGLEVEL \
+    -O fair \
+    -c $CONCURRENCY \
+    -E \
+    --max-tasks-per-child=50

bin/docker_start.sh

@@ -2,24 +2,20 @@
 
 set -ex
 
-# Wait for the database container
+# Figure out abspath of this script
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+# wait for required services
 # See: https://docs.docker.com/compose/startup-order/
-export PGHOST=${DB_HOST:-db}
-export PGPORT=${DB_PORT:-5432}
+${SCRIPTPATH}/wait_for_db.sh
 
-fixtures_dir=${FIXTURES_DIR:-/app/fixtures}
+# fixtures_dir=${FIXTURES_DIR:-/app/fixtures}
 
 uwsgi_port=${UWSGI_PORT:-8000}
 uwsgi_processes=${UWSGI_PROCESSES:-4}
 uwsgi_threads=${UWSGI_THREADS:-1}
 
-until pg_isready; do
-  >&2 echo "Waiting for database connection..."
-  sleep 1
-done
-
->&2 echo "Database is up."
-
 # Apply database migrations
 >&2 echo "Apply database migrations"
 python src/manage.py migrate

bin/setup_configuration.sh

@@ -3,7 +3,12 @@
 # setup initial configuration using environment variables
 # Run this script from the root of the repository
 
-#set -e
+set -e
+
+# Figure out abspath of this script
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
 ${SCRIPTPATH}/wait_for_db.sh
 
 src/manage.py migrate

bin/wait_for_db.sh

@@ -12,4 +12,4 @@ until pg_isready; do
   sleep 1
 done
 
->&2 echo "Database is up."
+>&2 echo "Database is up."

docker-compose.yml

@@ -1,5 +1,17 @@
-# Inspired by https://docs.docker.com/compose/django/
-version: '3'
+#
+# Docker-compose for full Open Inwoner stack.
+#
+# DISCLAIMER: THIS IS FOR DEVELOPMENT PURPOSES ONLY AND NOT SUITABLE FOR PRODUCTION.
+#
+# You can use this docker-compose to spin up a local Open Inwoner stack for demo/try-out
+# purposes, or to get some insight in the various components involved (e.g. to build
+# your Helm charts from). Note that various environment variables are UNSAFE and merely
+# specified so that you can get up and running with the least amount of friction.
+#
+# Before deploying to production, please review the environment configuration reference:
+# https://open-inwoner.readthedocs.io/
+#
+version: '3.4'
 
 services:
   db:
@@ -44,6 +56,8 @@ services:
   web:
     build: &web_build
       context: .
+      args:
+        RELEASE: ${TAG:-latest}
     container_name: open-inwoner-web
     image: maykinmedia/open-inwoner:${TAG:-latest}
     environment: &web_env
@@ -60,9 +74,22 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
       - CELERY_LOGLEVEL=DEBUG
       - ES_HOST=elasticsearch
+      # Needed for Celery Flower to match the TIME_ZONE configured in the
+      # settings used by workers and beat containers.
+      - TZ=Europe/Amsterdam
+      # WARNING: Strictly for development!
       - DISABLE_2FA=${DISABLE_2FA:-True}
       - DEBUG=True
       - IS_HTTPS=no
+    healthcheck:
+      test: ["CMD", "python", "-c", "import requests; exit(requests.head('http://localhost:8000/admin/').status_code not in [200, 302])"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      # This should allow for enough time for migrations to run before the max
+      # retries have passed. This healthcheck in turn allows other containers
+      # to wait for the database migrations.
+      start_period: 60s
     volumes: &web_volumes
       - media:/app/media
       - private_media:/app/private_media
@@ -91,10 +118,44 @@ services:
     image: maykinmedia/open-inwoner:${TAG:-latest}
     environment: *web_env
     command: /celery_worker.sh
+    healthcheck:
+      test: ["CMD", "python", "/app/bin/check_celery_worker_liveness.py"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
     volumes: *web_volumes
     depends_on:
-      - db
-      - redis
+      web:
+        # This health check condition is needed because Celery Beat will 
+        # try to convert the CELERY_BEAT_SCHEDULE into database entries. For 
+        # this, migrations need to be finished. If Celery tasks were still 
+        # pending, the database also needs to be ready for Celery itself. We 
+        # therefore have the health check here, and make Celery beat and 
+        # monitor containers depend on the celery container.
+        condition: service_healthy
+      redis:
+        condition: service_started
+    networks:
+      - openinwoner-dev
+
+  celery-beat:
+    build: *web_build
+    image: maykinmedia/open-inwoner:${TAG:-latest}
+    environment: *web_env
+    command: /celery_beat.sh
+    depends_on:
+      - celery
+    networks:
+      - openinwoner-dev
+
+  celery-monitor:
+    build: *web_build
+    image: maykinmedia/open-inwoner:${TAG:-latest}
+    environment: *web_env
+    command: /celery_monitor.sh
+    depends_on:
+      - celery
     networks:
       - openinwoner-dev
 

src/open_inwoner/celery.py

@@ -1,7 +1,9 @@
+from pathlib import Path
+
 from django.conf import settings
 
-from celery import Celery
-from celery.signals import setup_logging
+from celery import Celery, bootsteps
+from celery.signals import setup_logging, worker_ready, worker_shutdown
 
 from .setup import setup_env
 
@@ -26,3 +28,45 @@ def config_loggers(*args, **kwargs):
     from logging.config import dictConfig
 
     dictConfig(settings.LOGGING)
+
+
+HEARTBEAT_FILE = Path(settings.BASE_DIR) / "tmp" / "celery_worker_heartbeat"
+READINESS_FILE = Path(settings.BASE_DIR) / "tmp" / "celery_worker_ready"
+
+
+#
+# Utilities for checking the health of celery workers
+#
+class LivenessProbe(bootsteps.StartStopStep):
+    requires = {"celery.worker.components:Timer"}
+
+    def __init__(self, worker, **kwargs):
+        self.requests = []
+        self.tref = None
+
+    def start(self, worker):
+        self.tref = worker.timer.call_repeatedly(
+            10.0,
+            self.update_heartbeat_file,
+            (worker,),
+            priority=10,
+        )
+
+    def stop(self, worker):
+        HEARTBEAT_FILE.unlink(missing_ok=True)
+
+    def update_heartbeat_file(self, worker):
+        HEARTBEAT_FILE.touch()
+
+
+@worker_ready.connect
+def worker_ready(**_):
+    READINESS_FILE.touch()
+
+
+@worker_shutdown.connect
+def worker_shutdown(**_):
+    READINESS_FILE.unlink(missing_ok=True)
+
+
+app.steps["worker"].add(LivenessProbe)