Feature/tc local helm (#26)

* Add helm chart
* Custom package for project branches

.github/workflows/ci.yml

@@ -1,10 +1,23 @@
 name: Run test
 
 on:
-  push:
-    branches:
-      - develop
+  workflow_call:
+    inputs:
+      push_docker_image:
+        type: string  # true or false
+        default: "false"
+    outputs:
+      docker_image_name:
+        description: "Only docker image name"
+        value: ${{ jobs.test.outputs.docker_image_name }}
+      docker_image_tag:
+        description: "Only docker image tag"
+        value: ${{ jobs.test.outputs.docker_image_tag }}
+      docker_image:
+        description: "docker image with tag"
+        value: ${{ jobs.test.outputs.docker_image }}
   pull_request:
+  # NOTE: For other, they should be run through helm github action ./helm-publish.yml
 
 env:
   DJANGO_SECRET_KEY: "ci-test-insecure-django-secret-key"
@@ -14,17 +27,45 @@ jobs:
   test:
     name: 🚴 Test 🚴
     runs-on: ubuntu-latest
+    outputs:
+      docker_image_name: ${{ steps.prep.outputs.tagged_image_name }}
+      docker_image_tag: ${{ steps.prep.outputs.tag }}
+      docker_image: ${{ steps.prep.outputs.tagged_image }}
 
     steps:
       - uses: actions/checkout@main
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: ${{ inputs.push_docker_image }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: 🐳 Prepare Docker
         id: prep
+        env:
+          IMAGE_NAME: ghcr.io/${{ github.repository }}
+          PUSH_DOCKER_IMAGE: ${{ inputs.push_docker_image }}
         run: |
-          TAG=$(echo $GITHUB_SHA | head -c7)
-          IMAGE="backend"
-          echo "tagged_image=${IMAGE}:${TAG}" >> $GITHUB_OUTPUT
+          BRANCH_NAME=$(echo $GITHUB_REF_NAME | sed 's|:|-|' | tr '[:upper:]' '[:lower:]' | sed 's/_/-/g' | cut -c1-100 | sed 's/-*$//')
+
+          # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
+          if [[ "$BRANCH_NAME" == *"/"* ]]; then
+              # XXX: Change the docker image package to -beta
+              IMAGE_NAME="$IMAGE_NAME-beta"
+              TAG="$(echo "$BRANCH_NAME" | sed 's|/|-|g').$(echo $GITHUB_SHA | head -c7)"
+          else
+              TAG="$BRANCH_NAME.$(echo $GITHUB_SHA | head -c7)"
+          fi
+
+          echo "tagged_image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
           echo "tag=${TAG}" >> $GITHUB_OUTPUT
+          echo "tagged_image=${IMAGE_NAME}:${TAG}" >> $GITHUB_OUTPUT
+          echo "push_docker_image=$PUSH_DOCKER_IMAGE"  >> $GITHUB_OUTPUT
+          echo "::notice::Tagged docker image: ${IMAGE_NAME}:${TAG}"
+
       - name: 🐳 Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -44,7 +85,7 @@ jobs:
           context: .
           builder: ${{ steps.buildx.outputs.name }}
           file: Dockerfile
-          push: false
+          push: ${{ steps.prep.outputs.push_docker_image == 'true' }}
           load: true
           tags: ${{ steps.prep.outputs.tagged_image }}
           cache-from: type=local,src=/tmp/.buildx-cache
@@ -63,3 +104,19 @@ jobs:
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  validate_helm:
+    name: 🚴 Validate Helm 🚴
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@main
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: 🐳 Helm lint
+        run: helm lint ./helm --values ./helm/values-test.yaml
+
+      - name: 🐳 Helm template
+        run: helm template ./helm --values ./helm/values-test.yaml

.github/workflows/helm-publish.yml

@@ -0,0 +1,75 @@
+name: Builds and pushes Docker Images and Helm charts to Github Registry
+
+on:
+  push:
+    branches:
+      - develop
+      - project/*
+    # XXX: To add tags: Update the -beta logic
+
+permissions:
+  packages: write
+
+jobs:
+  ci:
+    name: CI
+    uses: ./.github/workflows/ci.yml
+    with:
+      push_docker_image: true
+
+  build:
+    name: Publish Helm
+    needs: ci
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Install Helm
+      uses: azure/setup-helm@v3
+
+    - name: Tag docker image in Helm Chart values.yaml
+      env:
+        IMAGE_NAME: ${{ needs.ci.outputs.docker_image_name }}
+        IMAGE_TAG: ${{ needs.ci.outputs.docker_image_tag }}
+      run: |
+        # Update values.yaml with latest docker image
+        sed -i "s|SET-BY-CICD-IMAGE|$IMAGE_NAME|" helm/values.yaml
+        sed -i "s/SET-BY-CICD-TAG/$IMAGE_TAG/" helm/values.yaml
+
+    - name: Package Helm Chart
+      id: set-variables
+      run: |
+        # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
+        if [[ "$GITHUB_REF_NAME" == *"/"* ]]; then
+            # XXX: Change the helm chart to <chart-name>-beta
+            sed -i 's/^name: \(.*\)/name: \1-beta/' helm/Chart.yaml
+        fi
+
+        SHA_SHORT=$(git rev-parse --short HEAD)
+        sed -i "s/SET-BY-CICD/$SHA_SHORT/g" helm/Chart.yaml
+        helm package ./helm -d .helm-charts
+
+    - name: Push Helm Chart
+      env:
+        IMAGE: ${{ needs.ci.outputs.docker_image }}
+        OCI_REPO: oci://ghcr.io/${{ github.repository }}
+      run: |
+        PACKAGE_FILE=$(ls .helm-charts/*.tgz | head -n 1)
+        echo "# Helm Chart" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "Tagged Image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "Helm push output" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo '```bash' >> $GITHUB_STEP_SUMMARY
+        helm push "$PACKAGE_FILE" $OCI_REPO >> $GITHUB_STEP_SUMMARY
+        echo '```' >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 
 # editors
 .idea/
+
+# helm
+.helm-charts/

Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.12-slim-bullseye
 
 LABEL maintainer="Togglecorp Dev"
+LABEL org.opencontainers.image.source="https://github.com/toggle-corp/ai-chatbot-backend"
 
 ENV PYTHONUNBUFFERED=1
 

helm/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: ai-chatbot-backend-helm
+description: "Helm Chart to deploy the TC chatbot backend Infrastructure"
+type: application
+version: 0.0.1-SET-BY-CICD
+sources:
+  - https://github.com/toggle-corp/ai-chatbot-backend

helm/templates/_helpers.tpl

@@ -0,0 +1,34 @@
+{{/*
+    Expand the name of the chart.
+*/}}
+
+{{- define "tc-chatbot-backend.name" -}}
+    {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+    Create a default fully qualified app name.
+    We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+    If release name contains chart name it will be used as a full name.
+*/}}
+
+{{- define "tc-chatbot-backend.fullname" -}}
+    {{- if .Values.fullnameOverride -}}
+        {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+    {{- else -}}
+        {{- $name := default .Chart.Name .Values.nameOverride -}}
+        {{- if contains $name .Release.Name -}}
+            {{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+        {{- else -}}
+            {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+        {{- end -}}
+    {{- end -}}
+{{- end -}}
+
+{{/*
+    Create chart name and version as used by the chart label.
+*/}}
+
+{{- define "tc-chatbot-backend.chart" -}}
+    {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

helm/templates/api/deployment.yaml

@@ -0,0 +1,57 @@
+{{- if .Values.api.enabled }}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-api
+  labels:
+    component: api-deployment
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+    # XXX: Add global lables?
+spec:
+  replicas: {{ .Values.api.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "tc-chatbot-backend.name" . }}
+      release: {{ .Release.Name }}
+      run: {{ .Release.Name }}-api
+  template:
+    metadata:
+      annotations:
+        checksum/secret: {{ include (print .Template.BasePath "/config/secret.yaml") . | sha256sum }}
+        checksum/configmap: {{ include (print .Template.BasePath "/config/configmap.yaml") . | sha256sum }}
+      labels:
+        app: {{ template "tc-chatbot-backend.name" . }}
+        release: {{ .Release.Name }}
+        run: {{ .Release.Name }}-api
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}-api
+          image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["bash", "/code/scripts/run_prod.sh"]
+          ports:
+            - name: http
+              containerPort: {{ .Values.api.containerPort }}
+              protocol: TCP
+          livenessProbe: # FIXME: Fix Liveness Probe
+            httpGet:
+              path: / # TODO:
+              port: {{ .Values.api.containerPort }}
+            initialDelaySeconds: 10180  # TODO:?
+            periodSeconds: 5  # TODO:?
+          resources:
+            requests:
+              cpu: {{ .Values.api.resources.requests.cpu }}
+              memory: {{ .Values.api.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.api.resources.limits.cpu }}
+              memory: {{ .Values.api.resources.limits.memory }}
+          envFrom:
+            - secretRef:
+                name: {{ template "tc-chatbot-backend.fullname" . }}-api-secret
+            - configMapRef:
+                name: {{ template "tc-chatbot-backend.fullname" . }}-api-configmap
+
+{{- end }}

helm/templates/api/ingress.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.ingress.api.enabled }}
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-api-ingress
+  labels:
+    app: {{ template "tc-chatbot-backend.name" . }}
+    component: api-service
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  rules:
+    - host: {{ required "ingress.api.host" .Values.ingress.api.host | quote }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ template "tc-chatbot-backend.fullname" . }}-api-svc
+                port:
+                  number: 80
+
+{{- end }}

helm/templates/api/service.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.api.enabled -}}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-api-svc
+  labels:
+    app: {{ template "tc-chatbot-backend.name" . }}
+    component: api-service
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  type: ClusterIP
+  selector:
+    app: {{ template "tc-chatbot-backend.name" . }}
+    release: {{ .Release.Name }}
+    run: {{ .Release.Name }}-api
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: {{ .Values.api.containerPort }}
+      nodePort: null
+
+{{- end }}

helm/templates/config/configmap.yaml

@@ -0,0 +1,46 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ template "tc-chatbot-backend.fullname" . }}-api-configmap
+  labels:
+    component: api-deployment
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+data:
+  APP_ENVIRONMENT: {{ .Values.environment | upper | quote }}
+  APP_DOMAIN: {{ required "env.APP_DOMAIN" .Values.environment | upper | quote }}
+  DJANGO_TIME_ZONE: {{ .Values.env.DJANGO_TIME_ZONE }}
+  DJANGO_DEBUG: {{ .Values.env.DJANGO_DEBUG | quote }}
+  DJANGO_ALLOWED_HOST: {{ .Values.env.DJANGO_ALLOWED_HOST | quote }}
+  APP_HTTP_PROTOCOL: {{ .Values.env.APP_HTTP_PROTOCOL | quote }}
+
+  {{- if .Values.redis.enabled }}
+  # Redis
+  DJANGO_CACHE_REDIS_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/1"
+  # -- Celery
+  CELERY_REDIS_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
+  CELERY_BROKER_URL: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
+  CELERY_RESULT_BACKEND: "redis://{{ template "tc-chatbot-backend.fullname" . }}-redis:6379/0"
+  {{- else }}
+  CELERY_REDIS_URL: {{ required "env.CELERY_REDIS_URL" .Values.env.CELERY_REDIS_URL | quote }}
+  CACHE_REDIS_URL: {{ required "env.CACHE_REDIS_URL" .Values.env.CACHE_REDIS_URL | quote }}
+  {{- end }}
+
+  # Sentry
+  SENTRY_TRACES_SAMPLE_RATE: {{ .Values.env.SENTRY_TRACES_SAMPLE_RATE | quote }}
+  SENTRY_PROFILE_SAMPLE_RATE: {{ .Values.env.SENTRY_PROFILE_SAMPLE_RATE | quote }}
+  # App Domain
+  # Vector Database
+  QDRANT_DB_HOST: {{ .Values.env.QDRANT_DB_HOST | quote }}
+  QDRANT_DB_PORT: {{ .Values.env.QDRANT_DB_PORT | quote }}
+  QDRANT_DB_COLLECTION_NAME: {{ .Values.env.QDRANT_DB_COLLECTION_NAME | quote }}
+  # Embedding Model
+  EMBEDDING_MODEL_URL: {{ .Values.env.EMBEDDING_MODEL_URL | quote }}
+  EMBEDDING_MODEL_NAME: {{ .Values.env.EMBEDDING_MODEL_NAME | quote }}
+  EMBEDDING_MODEL_VECTOR_SIZE: {{ .Values.env.EMBEDDING_MODEL_VECTOR_SIZE | quote }}
+  EMBEDDING_MODEL_TYPE: {{ .Values.env.EMBEDDING_MODEL_TYPE | quote }}
+  OLLAMA_EMBEDDING_MODEL_BASE_URL: {{ .Values.env.OLLAMA_EMBEDDING_MODEL_BASE_URL | quote }}
+  # LLM Type
+  LLM_TYPE: {{ .Values.env.LLM_TYPE | quote }}
+  LLM_MODEL_NAME: {{ .Values.env.LLM_MODEL_NAME | quote }}
+  LLM_OLLAMA_BASE_URL: {{ .Values.env.LLM_OLLAMA_BASE_URL | quote }}