Feature/embedding model helm (#37)

* Change -beta to -alpha * Fix some typo issues * Add sub charts and embedding-model * Add Redis/Postgres/Qdrant as sub-chart * Fix helm dependency issue
toggle-corp · Nov 11, 2024 · f50928f · f50928f
1 parent ab91cb4
commit f50928f
Show file tree

Hide file tree

Showing 15 changed files with 247 additions and 84 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -47,23 +47,22 @@ jobs:
         id: prep
         env:
           IMAGE_NAME: ghcr.io/${{ github.repository }}
-          PUSH_DOCKER_IMAGE: ${{ inputs.push_docker_image }}
         run: |
           BRANCH_NAME=$(echo $GITHUB_REF_NAME | sed 's|:|-|' | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g' | cut -c1-100 | sed 's/-*$//')
 
           # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
           if [[ "$BRANCH_NAME" == *"/"* ]]; then
-              # XXX: Change the docker image package to -beta
-              IMAGE_NAME="$IMAGE_NAME-beta"
+              # XXX: Change the docker image package to -alpha
+              IMAGE_NAME="$IMAGE_NAME-alpha"
               TAG="$(echo "$BRANCH_NAME" | sed 's|/|-|g').$(echo $GITHUB_SHA | head -c7)"
           else
               TAG="$BRANCH_NAME.$(echo $GITHUB_SHA | head -c7)"
           fi
 
+          IMAGE_NAME=$(echo $IMAGE_NAME | tr '[:upper:]' '[:lower:]')
           echo "tagged_image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
           echo "tagged_image=${IMAGE_NAME}:${TAG}" >> $GITHUB_OUTPUT
-          echo "push_docker_image=$PUSH_DOCKER_IMAGE"  >> $GITHUB_OUTPUT
           echo "::notice::Tagged docker image: ${IMAGE_NAME}:${TAG}"
 
       - name: 🐳 Set up Docker Buildx
@@ -85,7 +84,7 @@ jobs:
           context: .
           builder: ${{ steps.buildx.outputs.name }}
           file: Dockerfile
-          push: ${{ steps.prep.outputs.push_docker_image == 'true' }}
+          push: false
           load: true
           tags: ${{ steps.prep.outputs.tagged_image }}
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -100,6 +99,13 @@ jobs:
             exit 1;
           }
 
+      - name: 🐳 Docker push
+        if: ${{ inputs.push_docker_image }}
+        uses: docker/build-push-action@v6
+        with:
+          tags: ${{ steps.prep.outputs.tagged_image }}
+          push: true
+
       - name: 🐳 Move docker cache (🧙 Hack fix)
         run: |
           rm -rf /tmp/.buildx-cache
@@ -115,6 +121,11 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v4
 
+      - name: 🐳 Helm dependency
+        run: |
+          yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock  | sh --
+          helm dependency build ./helm
+
       - name: 🐳 Helm lint
         run: helm lint ./helm --values ./helm/values-test.yaml
 

diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml
@@ -5,7 +5,7 @@ on:
     branches:
       - develop
       - project/*
-    # XXX: To add tags: Update the -beta logic
+    # XXX: To add tags: Update the -alpha logic
 
 permissions:
   packages: write
@@ -36,6 +36,11 @@ jobs:
     - name: Install Helm
       uses: azure/setup-helm@v3
 
+    - name: 🐳 Helm dependency
+      run: |
+        yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock  | sh --
+        helm dependency build ./helm
+
     - name: Tag docker image in Helm Chart values.yaml
       env:
         IMAGE_NAME: ${{ needs.ci.outputs.docker_image_name }}
@@ -50,8 +55,8 @@ jobs:
       run: |
         # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
         if [[ "$GITHUB_REF_NAME" == *"/"* ]]; then
-            # XXX: Change the helm chart to <chart-name>-beta
-            sed -i 's/^name: \(.*\)/name: \1-beta/' helm/Chart.yaml
+            # XXX: Change the helm chart to <chart-name>-alpha
+            sed -i 's/^name: \(.*\)/name: \1-alpha/' helm/Chart.yaml
         fi
 
         SHA_SHORT=$(git rev-parse --short HEAD)

diff --git a/content/models.py b/content/models.py
@@ -1,6 +1,6 @@
 import uuid
 
-from django.db import models
+from django.db import models, transaction
 from django.utils.translation import gettext_lazy as _
 
 from common.models import UserResource
@@ -50,4 +50,4 @@ def save(self, *args, **kwargs):
 
         super().save(*args, **kwargs)
         if self.document_status == self.DocumentStatus.TEXT_EXTRACTED:
-            create_embedding_for_content_task(self.id)
+            transaction.on_commit(lambda: create_embedding_for_content_task.delay(self.id))
diff --git a/helm/.gitignore b/helm/.gitignore
@@ -0,0 +1,2 @@
+values-local.yaml
+charts
diff --git a/helm/Chart.lock b/helm/Chart.lock
@@ -0,0 +1,12 @@
+dependencies:
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 20.2.1
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 16.1.2
+- name: qdrant
+  repository: https://qdrant.github.io/qdrant-helm
+  version: 1.12.2
+digest: sha256:3b94766cdd43351699a96ec2bebc27656e34de13c49c7be279db63f750cde6ae
+generated: "2024-11-10T23:44:51.273569466+05:45"
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
@@ -5,3 +5,23 @@ type: application
 version: 0.0.1-SET-BY-CICD
 sources:
   - https://github.com/toggle-corp/ai-chatbot-backend
+
+
+dependencies:
+  - name: redis
+    alias: redis
+    version: "20.2.1"
+    repository: "https://charts.bitnami.com/bitnami"
+    condition: redis.enabled
+
+  - name: postgresql
+    alias: postgresql
+    version: "16.1.2"
+    repository: "https://charts.bitnami.com/bitnami"
+    condition: postgresql.enabled
+
+  - name: qdrant
+    alias: qdrant
+    version: "1.12.2"
+    repository: "https://qdrant.github.io/qdrant-helm"
+    condition: qdrant.enabled
diff --git a/helm/templates/api/service.yaml b/helm/templates/api/service.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.api.enabled -}}
+{{- if .Values.api.enabled }}
 
 apiVersion: v1
 kind: Service

diff --git a/helm/templates/config/configmap.yaml b/helm/templates/config/configmap.yaml
@@ -15,27 +15,40 @@ data:
   APP_HTTP_PROTOCOL: {{ .Values.env.APP_HTTP_PROTOCOL | quote }}
 
   {{- if .Values.redis.enabled }}
-  # Redis
-  DJANGO_CACHE_REDIS_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/1"
-  # -- Celery
-  CELERY_REDIS_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
-  CELERY_BROKER_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
-  CELERY_RESULT_BACKEND: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
+  # Celery
+  CELERY_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0"
+  CELERY_BROKER_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0"
+  CELERY_RESULT_BACKEND: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/0"
+  # Cache
+  DJANGO_CACHE_REDIS_URL: "redis://{{ printf "%s-master" (include "common.names.fullname" .Subcharts.redis) }}:6379/1"
   {{- else }}
-  CELERY_REDIS_URL: {{ required "env.CELERY_REDIS_URL" .Values.env.CELERY_REDIS_URL | quote }}
+  # Celery
+  CELERY_BROKER_URL: {{ required "env.CELERY_BROKER_URL" .Values.env.CELERY_BROKER_URL | quote }}
   CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }}
+  CELERY_RESULT_BACKEND: {{ required "env.CELERY_RESULT_BACKEND" .Values.env.CELERY_RESULT_BACKEND | quote }}
+  # Cache
+  DJANGO_CACHE_REDIS_URL: {{ required "env.DJANGO_CACHE_REDIS_URL" .Values.env.DJANGO_CACHE_REDIS_URL | quote }}
   {{- end }}
 
   # Sentry
   SENTRY_TRACES_SAMPLE_RATE: {{ .Values.env.SENTRY_TRACES_SAMPLE_RATE | quote }}
   SENTRY_PROFILE_SAMPLE_RATE: {{ .Values.env.SENTRY_PROFILE_SAMPLE_RATE | quote }}
-  # App Domain
+
   # Vector Database
-  QDRANT_DB_HOST: {{ .Values.env.QDRANT_DB_HOST | quote }}
-  QDRANT_DB_PORT: {{ .Values.env.QDRANT_DB_PORT | quote }}
-  QDRANT_DB_COLLECTION_NAME: {{ .Values.env.QDRANT_DB_COLLECTION_NAME | quote }}
+  {{- if .Values.redis.enabled }}
+  QDRANT_DB_HOST: {{ include "qdrant.fullname" .Subcharts.qdrant | quote }}
+  {{- else }}
+  QDRANT_DB_HOST: {{ required "env.QDRANT_DB_HOST" .Values.env.QDRANT_DB_HOST | quote }}
+  {{- end }}
+  QDRANT_DB_PORT: {{ required "env.QDRANT_DB_PORT" .Values.env.QDRANT_DB_PORT | quote }}
+  QDRANT_DB_COLLECTION_NAME: {{ required "env.QDRANT_DB_COLLECTION_NAME" .Values.env.QDRANT_DB_COLLECTION_NAME | quote }}
+
   # Embedding Model
-  EMBEDDING_MODEL_URL: {{ .Values.env.EMBEDDING_MODEL_URL | quote }}
+  {{- if .Values.embeddingModel.enabled }}
+  EMBEDDING_MODEL_URL: http://{{ template "tc-chatbot-backend.fullname" . }}-embedding-model-svc
+  {{- else }}
+  EMBEDDING_MODEL_URL: {{ required ".Values.env.EMBEDDING_MODEL_URL" .Values.env.EMBEDDING_MODEL_URL | quote }}
+  {{- end }}
   EMBEDDING_MODEL_NAME: {{ .Values.env.EMBEDDING_MODEL_NAME | quote }}
   EMBEDDING_MODEL_VECTOR_SIZE: {{ .Values.env.EMBEDDING_MODEL_VECTOR_SIZE | quote }}
   EMBEDDING_MODEL_TYPE: {{ .Values.env.EMBEDDING_MODEL_TYPE | quote }}

diff --git a/helm/templates/config/secret.yaml b/helm/templates/config/secret.yaml
@@ -9,12 +9,22 @@ metadata:
 type: Opaque
 stringData:
   DJANGO_SECRET_KEY: {{ required "secrets.DJANGO_SECRET_KEY" .Values.secrets.DJANGO_SECRET_KEY | quote }}
+
   # Database
+  {{- if .Values.postgresql.enabled }}
+  DATABASE_HOST: {{ include "postgresql.v1.primary.fullname" .Subcharts.postgresql }}
+  DATABASE_PORT: {{ include "postgresql.v1.service.port" .Subcharts.postgresql | quote }}
+  DATABASE_USER: {{ .Values.postgresql.auth.username | quote }}
+  DATABASE_PASSWORD: {{ .Values.postgresql.auth.password | quote }}
+  DATABASE_NAME: {{ .Values.postgresql.auth.database | quote }}
+  {{- else }}
   DATABASE_NAME: {{ required "secrets.DATABASE_NAME" .Values.secrets.DATABASE_NAME | quote }}
   DATABASE_USER: {{ required "secrets.DATABASE_USER" .Values.secrets.DATABASE_USER | quote }}
   DATABASE_PASSWORD: {{ required "secrets.DATABASE_PASSWORD" .Values.secrets.DATABASE_PASSWORD | quote }}
   DATABASE_HOST: {{ required "secrets.DATABASE_HOST" .Values.secrets.DATABASE_HOST | quote }}
   DATABASE_PORT: {{ required "secrets.DATABASE_PORT" .Values.secrets.DATABASE_PORT | quote }}
+  {{- end }}
+
   # Sentry
   SENTRY_DSN: {{ .Values.secrets.SENTRY_DSN }}
   # OpenAI

diff --git a/helm/templates/embedding_model/deployment.yaml b/helm/templates/embedding_model/deployment.yaml
@@ -0,0 +1,59 @@
+{{- if .Values.embeddingModel.enabled }}
+
+# TODO: Add NODE SELECTOR
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-embedding-model
+  labels:
+    component: embedding-model-deployment
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+    # XXX: Add global lables?
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ template "tc-chatbot-backend.name" . }}
+      release: {{ .Release.Name }}
+      run: {{ .Release.Name }}-embedding-model
+  template:
+    metadata:
+      labels:
+        app: {{ template "tc-chatbot-backend.name" . }}
+        release: {{ .Release.Name }}
+        run: {{ .Release.Name }}-embedding-model
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}-embedding-model
+          image: "{{ .Values.embeddingModel.image.name }}:{{ .Values.embeddingModel.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["uvicorn", "app:app", "--host=0.0.0.0", "--port={{ .Values.embeddingModel.containerPort }}"]
+          ports:
+            - name: http
+              containerPort: {{ .Values.embeddingModel.containerPort }}
+              protocol: TCP
+          # nodeSelector: TODO:
+          #   nvidia: ssd
+          livenessProbe: # FIXME: Fix Liveness Probe
+            httpGet:
+              path: /  # TODO
+              port: {{ .Values.embeddingModel.containerPort }}
+            initialDelaySeconds: 10180  # TODO:?
+            periodSeconds: 5  # TODO:?
+          resources:
+            requests:
+              cpu: {{ .Values.embeddingModel.resources.requests.cpu }}
+              memory: {{ .Values.embeddingModel.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.embeddingModel.resources.limits.cpu }}
+              memory: {{ .Values.embeddingModel.resources.limits.memory }}
+          volumeMounts:
+          - mountPath: "/opt/models"
+            name: downloaded-models
+      volumes:
+        - name: downloaded-models
+          persistentVolumeClaim:
+            claimName: {{ template "tc-chatbot-backend.fullname" . }}-embedding-model-pvc
+
+{{- end }}
diff --git a/helm/templates/embedding_model/pvc.yaml b/helm/templates/embedding_model/pvc.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.embeddingModel.enabled }}
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-embedding-model-pvc
+  labels:
+    component: embedding-model-deployment
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: {{ .Values.embeddingModel.persistence.storageClassName }}
+  resources:
+    requests:
+      storage: {{ .Values.embeddingModel.persistence.size }}
+
+{{- end }}
diff --git a/helm/templates/redis/service.yaml → helm/templates/embedding_model/service.yaml b/helm/templates/redis/service.yaml → helm/templates/embedding_model/service.yaml
@@ -1,24 +1,24 @@
-{{- if .Values.redis.enabled -}}
+{{- if .Values.embeddingModel.enabled }}
 
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "tc-chatbot-backend.fullname" . }}-redis
+  name: {{ template "tc-chatbot-backend.fullname" . }}-embedding-model-svc
   labels:
     app: {{ template "tc-chatbot-backend.name" . }}
-    component: redis-service
+    component: embedding-model-service
     environment: {{ .Values.environment }}
     release: {{ .Release.Name }}
 spec:
   type: ClusterIP
-  ports:
-    - protocol: TCP
-      port: 6379
-      targetPort: 6379
-      nodePort: null
   selector:
     app: {{ template "tc-chatbot-backend.name" . }}
     release: {{ .Release.Name }}
-    run: {{ .Release.Name }}-redis
+    run: {{ .Release.Name }}-embedding-model
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: {{ .Values.embeddingModel.containerPort }}
+      nodePort: null
 
 {{- end }}
diff --git a/helm/templates/redis/deployment.yaml b/helm/templates/redis/deployment.yaml
diff --git a/helm/templates/streamlit/service.yaml b/helm/templates/streamlit/service.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.streamlit.enabled -}}
+{{- if .Values.streamlit.enabled }}
 
 apiVersion: v1
 kind: Service