diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml index 6edeffe..2ee4e54 100644 --- a/.github/workflows/helm-publish.yml +++ b/.github/workflows/helm-publish.yml @@ -5,7 +5,7 @@ on: branches: - develop - project/* - # XXX: To add tags: Update the -alpha logic + # XXX: To add tags: Update the -alpha logic permissions: packages: write diff --git a/helm/templates/argo-hooks/hook-job.yaml b/helm/templates/argo-hooks/hook-job.yaml new file mode 100644 index 0000000..57ee4a8 --- /dev/null +++ b/helm/templates/argo-hooks/hook-job.yaml @@ -0,0 +1,43 @@ +{{- range $hookName, $hook := .Values.argoHooks }} + +{{- if $hook.enabled }} + +--- +apiVersion: batch/v1 +kind: Job +metadata: + generateName: {{ template "tc-chatbot-backend.fullname" $ }}-{{ $hookName }}- + annotations: + argocd.argoproj.io/hook: {{ $hook.hook }} +spec: + template: + metadata: + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/config/secret.yaml") $ | sha256sum }} + checksum/configmap: {{ include (print $.Template.BasePath "/config/configmap.yaml") $ | sha256sum }} + spec: + restartPolicy: "Never" + containers: + - name: {{ $.Chart.Name }}-{{ $hookName }} + image: "{{ $.Values.image.name }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + command: + {{- range $hook.command }} + - "{{ . }}" + {{- end }} + resources: + requests: + cpu: {{ default $.Values.api.resources.requests.cpu $hook.requestsCpu }} + memory: {{ default $.Values.api.resources.requests.memory $hook.requestsMemory }} + limits: + cpu: {{ default $.Values.api.resources.limits.cpu $hook.limitsCpu }} + memory: {{ default $.Values.api.resources.limits.memory $hook.limitsMemory }} + envFrom: + - secretRef: + name: {{ template "tc-chatbot-backend.fullname" $ }}-api-secret + - configMapRef: + name: {{ template "tc-chatbot-backend.fullname" $ }}-api-configmap + +{{- end }} + +{{- end }} diff --git a/helm/templates/config/configmap.yaml b/helm/templates/config/configmap.yaml index c425b0b..9c44b96 100644 --- a/helm/templates/config/configmap.yaml +++ b/helm/templates/config/configmap.yaml @@ -57,3 +57,12 @@ data: LLM_TYPE: {{ .Values.env.LLM_TYPE | quote }} LLM_MODEL_NAME: {{ .Values.env.LLM_MODEL_NAME | quote }} LLM_OLLAMA_BASE_URL: {{ .Values.env.LLM_OLLAMA_BASE_URL | quote }} + + # S3 + {{- if .Values.env.USE_S3_BUCKET }} + USE_S3_BUCKET: {{ .Values.env.USE_S3_BUCKET | quote }} + AWS_S3_AWS_ENDPOINT_URL: {{ .Values.env.AWS_S3_AWS_ENDPOINT_URL | quote }} + AWS_S3_REGION: {{ required "env.AWS_S3_REGION" .Values.env.AWS_S3_REGION | quote }} + S3_STATIC_BUCKET_NAME: {{ required "env.S3_STATIC_BUCKET_NAME" .Values.env.S3_STATIC_BUCKET_NAME | quote }} + S3_MEDIA_BUCKET_NAME: {{ required "env.S3_MEDIA_BUCKET_NAME" .Values.env.S3_MEDIA_BUCKET_NAME | quote }} + {{- end }} diff --git a/helm/templates/config/secret.yaml b/helm/templates/config/secret.yaml index 508d0fb..8824d47 100644 --- a/helm/templates/config/secret.yaml +++ b/helm/templates/config/secret.yaml @@ -29,3 +29,9 @@ stringData: SENTRY_DSN: {{ .Values.secrets.SENTRY_DSN }} # OpenAI OPENAI_API_KEY: {{ .Values.secrets.OPENAI_API_KEY }} + + # S3 + {{- if .Values.env.USE_S3_BUCKET }} + AWS_S3_ACCESS_KEY_ID: {{ required ".secrets.AWS_S3_ACCESS_KEY_ID" .Values.secrets.AWS_S3_ACCESS_KEY_ID | quote }} + AWS_S3_SECRET_ACCESS_KEY: {{ required "secrets.AWS_S3_SECRET_ACCESS_KEY" .Values.secrets.AWS_S3_SECRET_ACCESS_KEY | quote }} + {{- end }} diff --git a/helm/values-test.yaml b/helm/values-test.yaml index dc87323..2d8252b 100644 --- a/helm/values-test.yaml +++ b/helm/values-test.yaml @@ -23,6 +23,9 @@ env: LLM_TYPE: test LLM_MODEL_NAME: test LLM_OLLAMA_BASE_URL: test.com + AWS_S3_REGION: us-east-x + S3_STATIC_BUCKET_NAME: bucket-x-static + S3_MEDIA_BUCKET_NAME: bucket-x-media secrets: DJANGO_SECRET_KEY: test @@ -36,3 +39,6 @@ secrets: SENTRY_DSN: test.com/1234 # OpenAI API key OPENAI_API_KEY: test + # S3 + AWS_S3_ACCESS_KEY_ID: access-key + AWS_S3_SECRET_ACCESS_KEY: access-secret diff --git a/helm/values.yaml b/helm/values.yaml index 3736d04..a59cc8a 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -57,6 +57,17 @@ qdrant: cpu: "100m" memory: 200Mi +argoHooks: + # NOTE: Make sure key are lowercase + database-migration: + enabled: true + hook: PostSync + command: ["./manage.py", "migrate"] + collect-static: + enabled: true + hook: PostSync + command: ["./manage.py", "collectstatic", "--noinput"] + # TODO: persistence for uploaded files api: enabled: true @@ -147,6 +158,12 @@ env: LLM_TYPE: 1 LLM_MODEL_NAME: LLM_OLLAMA_BASE_URL: + # S3 + USE_S3_BUCKET: true + AWS_S3_AWS_ENDPOINT_URL: + AWS_S3_REGION: + S3_STATIC_BUCKET_NAME: + S3_MEDIA_BUCKET_NAME: secrets: DJANGO_SECRET_KEY: @@ -160,3 +177,6 @@ secrets: SENTRY_DSN: # OpenAI API key OPENAI_API_KEY: + # S3 + AWS_S3_ACCESS_KEY_ID: + AWS_S3_SECRET_ACCESS_KEY: diff --git a/main/settings.py b/main/settings.py index f5da7b7..3ddbbf0 100644 --- a/main/settings.py +++ b/main/settings.py @@ -23,14 +23,23 @@ DJANGO_SECRET_KEY=str, DJANGO_CORS_ORIGIN_REGEX_WHITELIST=(list, []), DJANGO_ALLOWED_HOST=(list, ["*"]), - DJANGO_STATIC_ROOT=(str, os.path.join(BASE_DIR, "assets/static")), # Where to store - DJANGO_MEDIA_ROOT=(str, os.path.join(BASE_DIR, "assets/media")), # Where to store - DJANGO_STATIC_URL=(str, "/static/"), - DJANGO_MEDIA_URL=(str, "/media/"), DJANGO_TIME_ZONE=(str, "UTC"), APP_HTTP_PROTOCOL=str, APP_ENVIRONMENT=str, APP_DOMAIN=str, + # Storage + DJANGO_STATIC_ROOT=(str, os.path.join(BASE_DIR, "assets/static")), # Where to store + DJANGO_MEDIA_ROOT=(str, os.path.join(BASE_DIR, "assets/media")), # Where to store + DJANGO_STATIC_URL=(str, "/static/"), + DJANGO_MEDIA_URL=(str, "/media/"), + # -- S3 + USE_S3_BUCKET=(bool, False), + AWS_S3_AWS_ENDPOINT_URL=str, + AWS_S3_ACCESS_KEY_ID=str, + AWS_S3_SECRET_ACCESS_KEY=str, + AWS_S3_REGION=str, + S3_STATIC_BUCKET_NAME=str, + S3_MEDIA_BUCKET_NAME=str, # Database DATABASE_NAME=str, DATABASE_USER=str, @@ -225,11 +234,6 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ -STATIC_URL = env("DJANGO_STATIC_URL") -MEDIA_URL = env("DJANGO_MEDIA_URL") -STATIC_ROOT = env("DJANGO_STATIC_ROOT") -MEDIA_ROOT = env("DJANGO_MEDIA_ROOT") - # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -242,3 +246,37 @@ CELERY_TASK_SERIALIZER = env("CELERY_TASK_SERIALIZER") CELERY_RESULT_SERIALIZER = env("CELERY_RESULT_SERIALIZER") CELERY_TIMEZONE = env("CELERY_TIMEZONE") + + +STATIC_URL = env("DJANGO_STATIC_URL") +MEDIA_URL = env("DJANGO_MEDIA_URL") + +# Django storage +if env("USE_S3_BUCKET"): + AWS_S3_ENDPOINT_URL = env("AWS_S3_AWS_ENDPOINT_URL") + + AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID") + AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY") + AWS_S3_REGION_NAME = env("AWS_S3_REGION") + + STORAGES = { + "default": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "bucket_name": env("S3_MEDIA_BUCKET_NAME"), + "location": "media/", + "file_overwrite": False, + }, + }, + "staticfiles": { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": { + "bucket_name": env("S3_STATIC_BUCKET_NAME"), + "location": "static/", + }, + }, + } + +else: + STATIC_ROOT = env("DJANGO_STATIC_ROOT") + MEDIA_ROOT = env("DJANGO_MEDIA_ROOT") diff --git a/poetry.lock b/poetry.lock index c76f40a..dfab3c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -242,6 +242,44 @@ files = [ {file = "billiard-4.2.0.tar.gz", hash = "sha256:9a3c3184cb275aa17a732f93f65b20c525d3d9f253722d26a82194803ade5a2c"}, ] +[[package]] +name = "boto3" +version = "1.35.57" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.57-py3-none-any.whl", hash = "sha256:9edf49640c79a05b0a72f4c2d1e24dfc164344b680535a645f455ac624dc3680"}, + {file = "boto3-1.35.57.tar.gz", hash = "sha256:db58348849a5af061f0f5ec9c3b699da5221ca83354059fdccb798e3ddb6b62a"}, +] + +[package.dependencies] +botocore = ">=1.35.57,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.57" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.57-py3-none-any.whl", hash = "sha256:92ddd02469213766872cb2399269dd20948f90348b42bf08379881d5e946cc34"}, + {file = "botocore-1.35.57.tar.gz", hash = "sha256:d96306558085baf0bcb3b022d7a8c39c93494f031edb376694d2b2dcd0e81327"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.22.0)"] + [[package]] name = "celery" version = "5.4.0" @@ -558,6 +596,30 @@ files = [ [package.dependencies] django = ">=4.2" +[[package]] +name = "django-storages" +version = "1.14.4" +description = "Support for many storage backends in Django" +optional = false +python-versions = ">=3.7" +files = [ + {file = "django-storages-1.14.4.tar.gz", hash = "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f"}, + {file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"}, +] + +[package.dependencies] +boto3 = {version = ">=1.4.4", optional = true, markers = "extra == \"s3\""} +Django = ">=3.2" + +[package.extras] +azure = ["azure-core (>=1.13)", "azure-storage-blob (>=12)"] +boto3 = ["boto3 (>=1.4.4)"] +dropbox = ["dropbox (>=7.2.1)"] +google = ["google-cloud-storage (>=1.27)"] +libcloud = ["apache-libcloud"] +s3 = ["boto3 (>=1.4.4)"] +sftp = ["paramiko (>=1.15)"] + [[package]] name = "djangorestframework" version = "3.15.2" @@ -1082,6 +1144,17 @@ files = [ {file = "jiter-0.5.0.tar.gz", hash = "sha256:1d916ba875bcab5c5f7d927df998c4cb694d27dceddf3392e58beaf10563368a"}, ] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -2137,6 +2210,23 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "s3transfer" +version = "0.10.3" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d"}, + {file = "s3transfer-0.10.3.tar.gz", hash = "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "setuptools" version = "74.1.2" @@ -2552,4 +2642,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "07ee857017a6b3e1effd43b9d605940d40019412043d06b822a388d824c3b468" +content-hash = "927382c991963915a27733b32e328c101efa78df8227492bffdb38aa88fcb1c1" diff --git a/pyproject.toml b/pyproject.toml index e8bb2ab..37cc98b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ celery = "^5.4.0" redis = "^5.0.8" djangorestframework = "^3.15.2" gunicorn = "*" +django-storages = { version = "*", extras = ["s3"] } [build-system] requires = ["poetry-core"] diff --git a/scripts/run_prod.sh b/scripts/run_prod.sh index ae7b141..31ef4ef 100755 --- a/scripts/run_prod.sh +++ b/scripts/run_prod.sh @@ -1,7 +1,4 @@ #!/bin/bash -x -python manage.py collectstatic --noinput & -python manage.py migrate --noinput - gunicorn main.wsgi:application --bind 0.0.0.0:80 # gunicorn main.asgi:application --bind 0.0.0.0:80