From 8d9fc93f76e16d4d8b903c839fe1819c269683c8 Mon Sep 17 00:00:00 2001
From: Steven Bal <steven@maykinmedia.nl>
Date: Fri, 2 Feb 2024 10:46:09 +0100
Subject: [PATCH] :sparkles: Add celery for notifications

because celery is now required by notifications-api-common to send notifications
---
 Dockerfile                   |  3 +++
 bin/celery_worker.sh         | 34 ++++++++++++++++++++++++++++++++++
 bin/wait_for_db.sh           | 15 +++++++++++++++
 docker-compose.yml           | 20 +++++++++++++++++++-
 docs/installation/config.rst |  8 ++++++++
 src/objects/celery.py        | 12 ++++++++++++
 src/objects/conf/base.py     |  6 ++++++
 7 files changed, 97 insertions(+), 1 deletion(-)
 create mode 100755 bin/celery_worker.sh
 create mode 100755 bin/wait_for_db.sh
 create mode 100644 src/objects/celery.py

diff --git a/Dockerfile b/Dockerfile
index 2d5e1778..ebd531a7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,10 +47,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 COPY --from=backend-build /usr/local/lib/python3.10 /usr/local/lib/python3.10
 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
+COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
 
 # Stage 3.2 - Copy source code
 WORKDIR /app
 COPY ./bin/docker_start.sh /start.sh
+COPY ./bin/wait_for_db.sh /wait_for_db.sh
+COPY ./bin/celery_worker.sh /celery_worker.sh
 RUN mkdir /app/log /app/config
 
 # copy frontend build statics
diff --git a/bin/celery_worker.sh b/bin/celery_worker.sh
new file mode 100755
index 00000000..bb92e075
--- /dev/null
+++ b/bin/celery_worker.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+
+QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}}
+WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}}
+
+# Figure out abspath of this script
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+# wait for required services
+${SCRIPTPATH}/wait_for_db.sh
+
+# build up worker options array
+worker_options=(
+    "-Q$QUEUE"
+    "-n$WORKER_NAME"
+    "-l$LOGLEVEL"
+    "-Ofair"
+)
+
+if [[ -v CELERY_WORKER_CONCURRENCY ]]; then
+    echo "Using concurrency ${CELERY_WORKER_CONCURRENCY}"
+    worker_options+=( "-c${CELERY_WORKER_CONCURRENCY}" )
+fi
+
+echo "Starting celery worker $WORKER_NAME with queue $QUEUE"
+exec celery \
+    --app objects \
+    --workdir src \
+    worker "${worker_options[@]}"
diff --git a/bin/wait_for_db.sh b/bin/wait_for_db.sh
new file mode 100755
index 00000000..89e15e6a
--- /dev/null
+++ b/bin/wait_for_db.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+# Wait for the database container
+# See: https://docs.docker.com/compose/startup-order/
+export PGHOST=${DB_HOST:-db}
+export PGPORT=${DB_PORT:-5432}
+
+until pg_isready; do
+  >&2 echo "Waiting for database connection..."
+  sleep 1
+done
+
+>&2 echo "Database is up."
diff --git a/docker-compose.yml b/docker-compose.yml
index bfc00a80..f96bcbac 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,16 +9,34 @@ services:
       - POSTGRES_USER=${DB_USER:-objects}
       - POSTGRES_PASSWORD=${DB_PASSWORD:-objects}
 
+  redis:
+    image: redis
+
   web:
     build: .
-    environment:
+    environment: &app-env
       - DJANGO_SETTINGS_MODULE=objects.conf.docker
       - SECRET_KEY=${SECRET_KEY:-1(@f(-6s_u(5fd&1sg^uvu2s(c-9sapw)1era8q&)g)h@cwxxg}
       - OBJECTS_SUPERUSER_USERNAME=admin
       - OBJECTS_SUPERUSER_PASSWORD=admin
       - OBJECTS_SUPERUSER_EMAIL=admin@localhost
       - ALLOWED_HOSTS=*
+      - CACHE_DEFAULT=redis:6379/0
+      - CACHE_AXES=redis:6379/0
+      - CELERY_BROKER_URL=redis://redis:6379/1
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - CELERY_LOGLEVEL=DEBUG
+      - CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-4}
     ports:
       - 8000:8000
     depends_on:
       - db
+
+  celery:
+    build: .
+    image: maykinmedia/objects-api
+    environment: *app-env
+    command: /celery_worker.sh
+    depends_on:
+      - db
+      - redis
diff --git a/docs/installation/config.rst b/docs/installation/config.rst
index ae1d0d80..28334e71 100644
--- a/docs/installation/config.rst
+++ b/docs/installation/config.rst
@@ -90,6 +90,14 @@ Other settings
 * ``TWO_FACTOR_PATCH_ADMIN``: Whether to use the 2 Factor Authentication login flow for
   the admin or not. Default ``True``. You'll probably want to disable this when using OIDC.
 
+Celery
+-------
+
+* ``CELERY_BROKER_URL``: the URL of the broker that will be used to actually send the notifications (default: ``redis://localhost:6379/1``).
+
+* ``CELERY_RESULT_BACKEND``: the backend where the results of tasks will be stored (default: ``redis://localhost:6379/1``)
+
+
 Initial superuser creation
 --------------------------
 
diff --git a/src/objects/celery.py b/src/objects/celery.py
new file mode 100644
index 00000000..adc709a4
--- /dev/null
+++ b/src/objects/celery.py
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: EUPL-1.2
+# Copyright (C) 2022 Dimpact
+from celery import Celery
+
+from objects.setup import setup_env
+
+setup_env()
+
+app = Celery("objects")
+
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()
diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py
index 8661e827..0ba9a077 100644
--- a/src/objects/conf/base.py
+++ b/src/objects/conf/base.py
@@ -325,6 +325,12 @@
 # Library settings
 #
 
+#
+# CELERY
+#
+CELERY_BROKER_URL = config("CELERY_BROKER_URL", "redis://localhost:6379/1")
+CELERY_RESULT_BACKEND = config("CELERY_RESULT_BACKEND", "redis://localhost:6379/1")
+
 
 # Django-Admin-Index
 ADMIN_INDEX_SHOW_REMAINING_APPS_TO_SUPERUSERS = False