Fix/storage (#38)

* Add s3 integrations

- For alpha using minio

* Use template to generate argocd hooks

* Clean-up workflows

.github/workflows/helm-publish.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - develop
       - project/*
-    # XXX: To add tags: Update the -alpha logic
+      # XXX: To add tags: Update the -alpha logic
 
 permissions:
   packages: write

helm/templates/argo-hooks/hook-job.yaml

@@ -0,0 +1,43 @@
+{{- range $hookName, $hook := .Values.argoHooks }}
+
+{{- if $hook.enabled }}
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  generateName: {{ template "tc-chatbot-backend.fullname" $ }}-{{ $hookName }}-
+  annotations:
+    argocd.argoproj.io/hook: {{ $hook.hook }}
+spec:
+  template:
+    metadata:
+      annotations:
+        checksum/secret: {{ include (print $.Template.BasePath "/config/secret.yaml") $ | sha256sum }}
+        checksum/configmap: {{ include (print $.Template.BasePath "/config/configmap.yaml") $ | sha256sum }}
+    spec:
+      restartPolicy: "Never"
+      containers:
+        - name: {{ $.Chart.Name }}-{{ $hookName }}
+          image: "{{ $.Values.image.name }}:{{ $.Values.image.tag }}"
+          imagePullPolicy: {{ $.Values.image.pullPolicy }}
+          command:
+              {{- range $hook.command }}
+              - "{{ . }}"
+              {{- end }}
+          resources:
+            requests:
+              cpu: {{ default $.Values.api.resources.requests.cpu $hook.requestsCpu }}
+              memory: {{ default $.Values.api.resources.requests.memory $hook.requestsMemory }}
+            limits:
+              cpu: {{ default $.Values.api.resources.limits.cpu $hook.limitsCpu }}
+              memory: {{ default $.Values.api.resources.limits.memory $hook.limitsMemory }}
+          envFrom:
+            - secretRef:
+                name: {{ template "tc-chatbot-backend.fullname" $ }}-api-secret
+            - configMapRef:
+                name: {{ template "tc-chatbot-backend.fullname" $ }}-api-configmap
+
+{{- end }}
+
+{{- end }}

helm/templates/config/configmap.yaml

@@ -57,3 +57,12 @@ data:
   LLM_TYPE: {{ .Values.env.LLM_TYPE | quote }}
   LLM_MODEL_NAME: {{ .Values.env.LLM_MODEL_NAME | quote }}
   LLM_OLLAMA_BASE_URL: {{ .Values.env.LLM_OLLAMA_BASE_URL | quote }}
+
+  # S3
+  {{- if .Values.env.USE_S3_BUCKET }}
+  USE_S3_BUCKET: {{ .Values.env.USE_S3_BUCKET | quote }}
+  AWS_S3_AWS_ENDPOINT_URL: {{ .Values.env.AWS_S3_AWS_ENDPOINT_URL | quote }}
+  AWS_S3_REGION: {{ required "env.AWS_S3_REGION" .Values.env.AWS_S3_REGION | quote }}
+  S3_STATIC_BUCKET_NAME: {{ required "env.S3_STATIC_BUCKET_NAME" .Values.env.S3_STATIC_BUCKET_NAME | quote }}
+  S3_MEDIA_BUCKET_NAME: {{ required "env.S3_MEDIA_BUCKET_NAME" .Values.env.S3_MEDIA_BUCKET_NAME | quote }}
+  {{- end }}

helm/templates/config/secret.yaml

@@ -29,3 +29,9 @@ stringData:
   SENTRY_DSN: {{ .Values.secrets.SENTRY_DSN }}
   # OpenAI
   OPENAI_API_KEY: {{ .Values.secrets.OPENAI_API_KEY }}
+
+  # S3
+  {{- if .Values.env.USE_S3_BUCKET }}
+  AWS_S3_ACCESS_KEY_ID: {{ required ".secrets.AWS_S3_ACCESS_KEY_ID" .Values.secrets.AWS_S3_ACCESS_KEY_ID | quote }}
+  AWS_S3_SECRET_ACCESS_KEY: {{ required "secrets.AWS_S3_SECRET_ACCESS_KEY" .Values.secrets.AWS_S3_SECRET_ACCESS_KEY | quote }}
+  {{- end }}

helm/values-test.yaml

@@ -23,6 +23,9 @@ env:
   LLM_TYPE: test
   LLM_MODEL_NAME: test
   LLM_OLLAMA_BASE_URL: test.com
+  AWS_S3_REGION: us-east-x
+  S3_STATIC_BUCKET_NAME: bucket-x-static
+  S3_MEDIA_BUCKET_NAME: bucket-x-media
 
 secrets:
   DJANGO_SECRET_KEY: test
@@ -36,3 +39,6 @@ secrets:
   SENTRY_DSN: test.com/1234
   # OpenAI API key
   OPENAI_API_KEY: test
+  # S3
+  AWS_S3_ACCESS_KEY_ID: access-key
+  AWS_S3_SECRET_ACCESS_KEY: access-secret

helm/values.yaml

@@ -57,6 +57,17 @@ qdrant:
       cpu: "100m"
       memory: 200Mi
 
+argoHooks:
+  # NOTE: Make sure key are lowercase
+  database-migration:
+    enabled: true
+    hook: PostSync
+    command: ["./manage.py", "migrate"]
+  collect-static:
+    enabled: true
+    hook: PostSync
+    command: ["./manage.py", "collectstatic", "--noinput"]
+
 # TODO: persistence for uploaded files
 api:
   enabled: true
@@ -147,6 +158,12 @@ env:
   LLM_TYPE: 1
   LLM_MODEL_NAME:
   LLM_OLLAMA_BASE_URL:
+  # S3
+  USE_S3_BUCKET: true
+  AWS_S3_AWS_ENDPOINT_URL:
+  AWS_S3_REGION:
+  S3_STATIC_BUCKET_NAME:
+  S3_MEDIA_BUCKET_NAME:
 
 secrets:
   DJANGO_SECRET_KEY:
@@ -160,3 +177,6 @@ secrets:
   SENTRY_DSN:
   # OpenAI API key
   OPENAI_API_KEY:
+  # S3
+  AWS_S3_ACCESS_KEY_ID:
+  AWS_S3_SECRET_ACCESS_KEY:

main/settings.py

@@ -23,14 +23,23 @@
     DJANGO_SECRET_KEY=str,
     DJANGO_CORS_ORIGIN_REGEX_WHITELIST=(list, []),
     DJANGO_ALLOWED_HOST=(list, ["*"]),
-    DJANGO_STATIC_ROOT=(str, os.path.join(BASE_DIR, "assets/static")),  # Where to store
-    DJANGO_MEDIA_ROOT=(str, os.path.join(BASE_DIR, "assets/media")),  # Where to store
-    DJANGO_STATIC_URL=(str, "/static/"),
-    DJANGO_MEDIA_URL=(str, "/media/"),
     DJANGO_TIME_ZONE=(str, "UTC"),
     APP_HTTP_PROTOCOL=str,
     APP_ENVIRONMENT=str,
     APP_DOMAIN=str,
+    # Storage
+    DJANGO_STATIC_ROOT=(str, os.path.join(BASE_DIR, "assets/static")),  # Where to store
+    DJANGO_MEDIA_ROOT=(str, os.path.join(BASE_DIR, "assets/media")),  # Where to store
+    DJANGO_STATIC_URL=(str, "/static/"),
+    DJANGO_MEDIA_URL=(str, "/media/"),
+    # -- S3
+    USE_S3_BUCKET=(bool, False),
+    AWS_S3_AWS_ENDPOINT_URL=str,
+    AWS_S3_ACCESS_KEY_ID=str,
+    AWS_S3_SECRET_ACCESS_KEY=str,
+    AWS_S3_REGION=str,
+    S3_STATIC_BUCKET_NAME=str,
+    S3_MEDIA_BUCKET_NAME=str,
     # Database
     DATABASE_NAME=str,
     DATABASE_USER=str,
@@ -225,11 +234,6 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/5.1/howto/static-files/
 
-STATIC_URL = env("DJANGO_STATIC_URL")
-MEDIA_URL = env("DJANGO_MEDIA_URL")
-STATIC_ROOT = env("DJANGO_STATIC_ROOT")
-MEDIA_ROOT = env("DJANGO_MEDIA_ROOT")
-
 # Default primary key field type
 # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
 
@@ -242,3 +246,37 @@
 CELERY_TASK_SERIALIZER = env("CELERY_TASK_SERIALIZER")
 CELERY_RESULT_SERIALIZER = env("CELERY_RESULT_SERIALIZER")
 CELERY_TIMEZONE = env("CELERY_TIMEZONE")
+
+
+STATIC_URL = env("DJANGO_STATIC_URL")
+MEDIA_URL = env("DJANGO_MEDIA_URL")
+
+# Django storage
+if env("USE_S3_BUCKET"):
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_AWS_ENDPOINT_URL")
+
+    AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+    AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION")
+
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "bucket_name": env("S3_MEDIA_BUCKET_NAME"),
+                "location": "media/",
+                "file_overwrite": False,
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "bucket_name": env("S3_STATIC_BUCKET_NAME"),
+                "location": "static/",
+            },
+        },
+    }
+
+else:
+    STATIC_ROOT = env("DJANGO_STATIC_ROOT")
+    MEDIA_ROOT = env("DJANGO_MEDIA_ROOT")

pyproject.toml

@@ -23,6 +23,7 @@ celery = "^5.4.0"
 redis = "^5.0.8"
 djangorestframework = "^3.15.2"
 gunicorn = "*"
+django-storages = { version = "*", extras = ["s3"] }
 
 [build-system]
 requires = ["poetry-core"]

scripts/run_prod.sh

@@ -1,7 +1,4 @@
 #!/bin/bash -x
 
-python manage.py collectstatic --noinput &
-python manage.py migrate --noinput
-
 gunicorn main.wsgi:application --bind 0.0.0.0:80
 # gunicorn main.asgi:application --bind 0.0.0.0:80