fix(mail): use deferred mailer to get rid of synchronous exceptions d…

…uring SMTP timeout

charts/templates/_helpers.tpl

@@ -60,3 +60,10 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Component label
+*/}}
+{{- define "fiesta.componentLabels" -}}
+component: {{ . }}
+{{- end }}

charts/templates/web-deployment.yaml

@@ -4,16 +4,19 @@ metadata:
   name: web
   labels:
     {{- include "fiesta.labels" . | nindent 4 }}
+    {{- include "fiesta.componentLabels" "web" | nindent 4 }}
 spec:
   replicas: 1
   selector:
     matchLabels:
       {{- include "fiesta.selectorLabels" . | nindent 6 }}
+      {{- include "fiesta.componentLabels" "web" | nindent 6 }}
   template:
     metadata:
       labels:
         app: web
         {{- include "fiesta.selectorLabels" . | nindent 8 }}
+        {{- include "fiesta.componentLabels" "web" | nindent 8 }}
       annotations:
         kubectl.kubernetes.io/default-logs-container: web
     spec:

charts/templates/web-mailer.yaml

@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web-mailer
+  labels:
+    {{- include "fiesta.labels" . | nindent 4 }}
+    {{- include "fiesta.componentLabels" "mailer" | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "fiesta.selectorLabels" . | nindent 6 }}
+      {{- include "fiesta.componentLabels" "mailer" | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "fiesta.selectorLabels" . | nindent 8 }}
+        {{- include "fiesta.componentLabels" "mailer" | nindent 8 }}
+      annotations:
+        kubectl.kubernetes.io/default-logs-container: mailer
+    spec:
+      containers:
+        - name: mailer
+          image: "{{ .Values.web.repository }}:{{ .Values.web.tag | default .Chart.AppVersion }}"
+          command:
+            - /bin/sh
+            - -c
+            - ./manage.py runmailer_pg
+          envFrom:
+            - secretRef:
+                name: {{ .Values.web.secretName }}
+            - configMapRef:
+                name: {{ .Values.web.configName }}
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: web-mailer-retry
+  labels:
+    {{- include "fiesta.labels" . | nindent 4 }}
+    {{- include "fiesta.componentLabels" "mailer-retry" | nindent 4 }}
+spec:
+  schedule: "0,15,30,45 * * * *" #	Run every 15 minutes
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            {{- include "fiesta.selectorLabels" . | nindent 12 }}
+            {{- include "fiesta.componentLabels" "mailer-retry" | nindent 12 }}
+        spec:
+          containers:
+            - name: mailer-retry
+              image: "{{ .Values.web.repository }}:{{ .Values.web.tag | default .Chart.AppVersion }}"
+              command:
+                - /bin/sh
+                - -c
+                - ./manage.py retry_deferred
+              envFrom:
+                - secretRef:
+                    name: {{ .Values.web.secretName }}
+                - configMapRef:
+                    name: {{ .Values.web.configName }}
+          restartPolicy: OnFailure

fiesta/fiesta/settings/__init__.py

@@ -9,7 +9,7 @@
 from .db import DatabaseConfigMixin
 from .files import FilesConfigMixin, S3ConfigMixin
 from .logging import LoggingConfigMixin, SentryConfigMixin
-from .notifications import SmtpMailerConfigMixin
+from .notifications import DatabaseSmtpMailerConfigMixin
 from .project import ProjectConfigMixin
 from .security import SecurityConfigMixin
 from .templates import TemplatesConfigMixin
@@ -58,7 +58,7 @@ class LocalProduction(Base):
 
 
 class Production(
-    SmtpMailerConfigMixin,
+    DatabaseSmtpMailerConfigMixin,
     S3ConfigMixin,
     SentryConfigMixin,
     Base,

fiesta/fiesta/settings/notifications.py

@@ -5,7 +5,7 @@
 from ._utils import BaseConfigurationProtocol
 
 
-class SmtpMailerConfigMixin(BaseConfigurationProtocol):
+class DatabaseSmtpMailerConfigMixin(BaseConfigurationProtocol):
     MAILER_PRIMARY_BACKEND = Value(default="django.core.mail.backends.smtp.EmailBackend")
     MAILER_PRIMARY_TIMEOUT = PositiveIntegerValue(default=10, cast=int)
     MAILER_PRIMARY_HOST_USE_TLS = BooleanValue(default=True)
@@ -14,8 +14,15 @@ class SmtpMailerConfigMixin(BaseConfigurationProtocol):
     MAILER_PRIMARY_HOST_PASSWORD = SecretValue()
     MAILER_PRIMARY_HOST_USER = SecretValue()
 
+    # cannot use file, because pods
+    MAILER_USE_FILE_LOCK = BooleanValue(default=False)
+
+    # backend used by django itself
+    EMAIL_BACKEND = Value(default="mailer.backend.DbBackend")
+
+    # mailer used by db mailer to actually send emails
     @property
-    def EMAIL_BACKEND(self):
+    def MAILER_EMAIL_BACKEND(self):
         return self.MAILER_PRIMARY_BACKEND
 
     @property

fiesta/fiesta/settings/project.py

@@ -39,7 +39,8 @@ def ALLOWED_HOSTS(self):
         return [f".{self.ROOT_DOMAIN}"]
 
     # overwritten by production mixins
-    EMAIL_BACKEND = Value(default="django.core.mail.backends.console.EmailBackend")
+    EMAIL_BACKEND = Value(default="mailer.backend.DbBackend")
+    MAILER_EMAIL_BACKEND = Value(default="django.core.mail.backends.console.EmailBackend")
 
     def DEFAULT_FROM_EMAIL(self):
         return f"Fiesta+ <noreply@{self.ROOT_DOMAIN}>"
@@ -108,6 +109,8 @@ def DEFAULT_FROM_EMAIL(self):
         # health checks
         "health_check",
         "health_check.contrib.migrations",
+        # django-mailer
+        "mailer",
     ]
 
     MIDDLEWARE = [

pyproject.toml

@@ -53,6 +53,7 @@ sentry-sdk = {extras = ["django"], version = "^1.35.0"}
 django-admin-env-notice = "^1.0"
 cryptography = "41.0.7"
 django-admin-relation-links = "^0.2.5"
+django-mailer = "^2.3.1"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.17.0"