From dc0eea0c0fc86513a18cc88f46c2bd3d930d9d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Wed, 10 Apr 2024 08:42:44 +0200 Subject: [PATCH] feat(artifact): include artifact-backend (#464) Because - `artifact-backend` is now part of the Instill AI ecosystem and will be used by `api-gateway` and `model-backend` This commit - Includes `artifact-backend` in the `make latest` and `make all` commands and in the Helm charts --- .env | 7 + Dockerfile | 4 +- Makefile | 9 + charts/core/templates/_helpers.tpl | 17 ++ .../core/templates/api-gateway/configmap.yaml | 6 + .../templates/artifact-backend/configmap.yaml | 78 ++++++++ .../artifact-backend/deployment.yaml | 189 ++++++++++++++++++ .../core/templates/artifact-backend/hpa.yml | 33 +++ .../templates/artifact-backend/service.yaml | 22 ++ charts/core/values.yaml | 70 +++++++ docker-compose-build.yml | 23 +++ docker-compose-latest.yml | 27 +++ docker-compose.yml | 43 ++++ 13 files changed, 527 insertions(+), 1 deletion(-) create mode 100644 charts/core/templates/artifact-backend/configmap.yaml create mode 100644 charts/core/templates/artifact-backend/deployment.yaml create mode 100644 charts/core/templates/artifact-backend/hpa.yml create mode 100644 charts/core/templates/artifact-backend/service.yaml diff --git a/.env b/.env index cbea0c8c..33bf060a 100644 --- a/.env +++ b/.env @@ -78,6 +78,13 @@ MODEL_BACKEND_HOST=model-backend MODEL_BACKEND_PRIVATEPORT=3083 MODEL_BACKEND_PUBLICPORT=8083 +# artifact-backend +ARTIFACT_BACKEND_IMAGE=instill/artifact-backend +ARTIFACT_BACKEND_VERSION=0.2.1-alpha +ARTIFACT_BACKEND_HOST=artifact-backend +ARTIFACT_BACKEND_PRIVATEPORT=3085 +ARTIFACT_BACKEND_PUBLICPORT=8085 + # controller-model CONTROLLER_MODEL_IMAGE=instill/controller-model CONTROLLER_MODEL_VERSION=0.4.0-alpha diff --git a/Dockerfile b/Dockerfile index 61d03555..53e84401 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,7 @@ RUN git clone https://github.com/instill-ai/console.git RUN git clone https://github.com/instill-ai/pipeline-backend.git RUN git clone https://github.com/instill-ai/model-backend.git RUN git clone https://github.com/instill-ai/controller-model.git +RUN git clone https://github.com/instill-ai/artifact-backend.git FROM alpine:${ALPINE_VERSION} AS release @@ -47,10 +48,11 @@ RUN echo "Instill Core release codebase cloned on ${CACHE_DATE}" WORKDIR /instill-core -ARG API_GATEWAY_VERSION MGMT_BACKEND_VERSION CONSOLE_VERSION PIPELINE_BACKEND_VERSION MODEL_BACKEND_VERSION CONTROLLER_MODEL_VERSION +ARG API_GATEWAY_VERSION MGMT_BACKEND_VERSION CONSOLE_VERSION PIPELINE_BACKEND_VERSION MODEL_BACKEND_VERSION CONTROLLER_MODEL_VERSION ARTIFACT_BACKEND_VERSION RUN git clone -b v${API_GATEWAY_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/api-gateway.git RUN git clone -b v${MGMT_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/mgmt-backend.git RUN git clone -b v${CONSOLE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/console.git RUN git clone -b v${PIPELINE_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/pipeline-backend.git RUN git clone -b v${MODEL_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/model-backend.git RUN git clone -b v${CONTROLLER_MODEL_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/controller-model.git +RUN git clone -b v${ARTIFACT_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/artifact-backend.git diff --git a/Makefile b/Makefile index 45311098..fe849c38 100644 --- a/Makefile +++ b/Makefile @@ -96,6 +96,7 @@ build-latest: ## Build latest images for all Instill Core components PIPELINE_BACKEND_VERSION=latest \ MODEL_BACKEND_VERSION=latest \ CONTROLLER_MODEL_VERSION=latest \ + ARTIFACT_BACKEND_VERSION=latest \ CONSOLE_VERSION=latest \ COMPOSE_PROFILES=${PROFILE} docker compose -f docker-compose-build.yml build --progress plain \ "; \ @@ -113,6 +114,7 @@ build-release: ## Build release images for all Instill Core components --build-arg PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \ --build-arg MODEL_BACKEND_VERSION=${MODEL_BACKEND_VERSION} \ --build-arg CONTROLLER_MODEL_VERSION=${CONTROLLER_MODEL_VERSION} \ + --build-arg ARTIFACT_BACKEND_VERSION=${ARTIFACT_BACKEND_VERSION} \ --build-arg CONSOLE_VERSION=${CONSOLE_VERSION} \ --target release \ -t ${INSTILL_CORE_IMAGE_NAME}:${INSTILL_CORE_VERSION} . @@ -128,6 +130,7 @@ build-release: ## Build release images for all Instill Core components PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \ MODEL_BACKEND_VERSION=${MODEL_BACKEND_VERSION} \ CONTROLLER_MODEL_VERSION=${CONTROLLER_MODEL_VERSION} \ + ARTIFACT_BACKEND_VERSION=${ARTIFACT_BACKEND_VERSION} \ CONSOLE_VERSION=${CONSOLE_VERSION} \ COMPOSE_PROFILES=${PROFILE} docker compose -f docker-compose-build.yml build --progress plain \ "; \ @@ -213,6 +216,7 @@ helm-integration-test-latest: ## Run integration test on t --set itMode.enabled=true \ --set apiGateway.image.tag=latest \ --set mgmtBackend.image.tag=latest \ + --set artifactBackend.image.tag=latest \ --set pipelineBackend.image.tag=latest \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=latest \ @@ -254,6 +258,7 @@ helm-integration-test-release: ## Run integration test on --set itMode.enabled=true \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ + --set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \ --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \ @@ -333,6 +338,7 @@ ifeq ($(UNAME_S),Darwin) --set tags.prometheusStack=false \ --set apiGateway.image.tag=latest \ --set mgmtBackend.image.tag=latest \ + --set artifactBackend.image.tag=latest \ --set pipelineBackend.image.tag=latest \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=latest \ @@ -350,6 +356,7 @@ else ifeq ($(UNAME_S),Linux) --set tags.prometheusStack=false \ --set apiGateway.image.tag=latest \ --set mgmtBackend.image.tag=latest \ + --set artifactBackend.image.tag=latest \ --set pipelineBackend.image.tag=latest \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=latest \ @@ -411,6 +418,7 @@ ifeq ($(UNAME_S),Darwin) --set tags.prometheusStack=false \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ + --set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \ --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \ @@ -428,6 +436,7 @@ else ifeq ($(UNAME_S),Linux) --set tags.prometheusStack=false \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ + --set artifactBackend.image.tag=${ARTIFACT_BACKEND_VERSION} \ --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \ --set pipelineBackend.excludelocalconnector=false \ --set modelBackend.image.tag=${MODEL_BACKEND_VERSION} \ diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl index c250eb60..1cb42c9d 100644 --- a/charts/core/templates/_helpers.tpl +++ b/charts/core/templates/_helpers.tpl @@ -166,6 +166,23 @@ controller-model {{- end -}} {{/* +artifact-backend +*/}} +{{- define "core.artifactBackend" -}} +{{- printf "%s-artifact-backend" (include "core.fullname" .) -}} +{{- end -}} + +{{/* artifact-backend service and container public port */}} +{{- define "core.artifactBackend.publicPort" -}} +{{- printf "8085" -}} +{{- end -}} + +{{/* artifact-backend service and container private port */}} +{{- define "core.artifactBackend.privatePort" -}} +{{- printf "3085" -}} +{{- end -}} +{{/* + console */}} {{- define "core.console" -}} diff --git a/charts/core/templates/api-gateway/configmap.yaml b/charts/core/templates/api-gateway/configmap.yaml index efde1e40..3211c905 100644 --- a/charts/core/templates/api-gateway/configmap.yaml +++ b/charts/core/templates/api-gateway/configmap.yaml @@ -28,6 +28,12 @@ data: # model-backend MODEL_BACKEND_HOST={{ template "core.modelBackend" . }} MODEL_BACKEND_PUBLICPORT={{ template "core.modelBackend.publicPort" . }} + MODEL_BACKEND_PRIVATEPORT={{ template "core.modelBackend.privatePort" . }} + + # artifact-backend + ARTIFACT_BACKEND_HOST={{ template "core.artifactBackend" . }} + ARTIFACT_BACKEND_PUBLICPORT={{ template "core.artifactBackend.publicPort" . }} + ARTIFACT_BACKEND_PRIVATEPORT={{ template "core.artifactBackend.privatePort" . }} # influxdb INFLUXDB_HOST={{ template "core.influxdb" . }} diff --git a/charts/core/templates/artifact-backend/configmap.yaml b/charts/core/templates/artifact-backend/configmap.yaml new file mode 100644 index 00000000..dc1c445c --- /dev/null +++ b/charts/core/templates/artifact-backend/configmap.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "core.artifactBackend" . }} + labels: + {{- include "core.labels" . | nindent 4 }} + app.kubernetes.io/component: artifact-backend +data: + config.yaml: |+ + server: + privateport: {{ template "core.artifactBackend.privatePort" . }} + publicport: {{ template "core.artifactBackend.publicPort" . }} + edition: {{ .Values.edition }} + usage: + {{- toYaml .Values.usage | nindent 8 }} + debug: {{ ternary "true" "false" (eq (.Values.logLevel | upper) "DEBUG") }} + {{- if .Values.internalTLS.enabled }} + https: + cert: /etc/instill-ai/core/ssl/artifact/tls.crt + key: /etc/instill-ai/core/ssl/artifact/tls.key + {{- end }} + database: + username: {{ default (include "core.database.username" .) .Values.database.external.username }} + password: {{ default (include "core.database.rawPassword" .) .Values.database.external.password }} + host: {{ default (include "core.database.host" .) .Values.database.external.host }} + port: {{ default (include "core.database.port" .) .Values.database.external.port }} + {{- if .Values.database.external_replica }} + replica: + username: {{ .Values.database.external_replica.username | default "" }} + password: {{ .Values.database.external_replica.password | default "" }} + host: {{ .Values.database.external_replica.host | default "" }} + port: {{ .Values.database.external_replica.port | default "" }} + replicationtimeframe: 1 + {{- end }} + name: artifact + version: {{ .Values.artifactBackend.dbVersion }} + timezone: Etc/UTC + pool: + idleconnections: {{ .Values.database.maxIdleConns }} + maxconnections: {{ .Values.database.maxOpenConns }} + connlifetime: {{ .Values.database.maxConnLifeTime }} + cache: + redis: + redisoptions: + addr: {{ default (include "core.redis.addr" .) .Values.redis.external.addr }} + influxdb: + url: {{ .Values.influxdbCloud.url }} + token: {{ .Values.influxdbCloud.token }} + org: {{ .Values.influxdbCloud.organization }} + bucket: {{ .Values.influxdbCloud.bucket }} + flushinterval: 10 # In seconds for non-blocking batch mode + https: + cert: + key: + log: + external: {{ .Values.tags.observability }} + otelcollector: + host: {{ template "core.otel" . }} + port: {{ template "core.otel.port" . }} + openfga: + host: {{ template "core.openfga" . }} + port: 8080 + {{- if .Values.database.external_replica }} + replica: + host: {{ template "core.openfga" . }} + port: 8082 + replicationtimeframe: 1 + {{- end }} + temporal: + hostport: {{ default (printf "%s-frontend-headless:%s" (include "core.temporal" .) (include "core.temporal.frontend.grpcPort" .)) .Values.artifactBackend.temporal.hostPort }} + namespace: {{ default "artifact-backend" .Values.artifactBackend.temporal.namespace }} + ca: {{ default "" .Values.artifactBackend.temporal.ca }} + cert: {{ default "" .Values.artifactBackend.temporal.cert }} + key: {{ default "" .Values.artifactBackend.temporal.key }} + serverName: {{ default "" .Values.artifactBackend.temporal.serverName }} + registry: + host: {{ template "core.registry" . }} + port: {{ template "core.registry.port" . }} diff --git a/charts/core/templates/artifact-backend/deployment.yaml b/charts/core/templates/artifact-backend/deployment.yaml new file mode 100644 index 00000000..1d644cab --- /dev/null +++ b/charts/core/templates/artifact-backend/deployment.yaml @@ -0,0 +1,189 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "core.artifactBackend" . }} + labels: + {{- include "core.labels" . | nindent 4 }} + app.kubernetes.io/component: artifact-backend + annotations: + rollme: {{ randAlphaNum 5 | quote }} +spec: + strategy: + type: {{ .Values.updateStrategy.type }} + {{- if eq .Values.updateStrategy.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ .Values.updateStrategy.rollingUpdate.maxSurge }} + maxUnavailable: {{ .Values.updateStrategy.rollingUpdate.maxUnavailable }} + {{- else}} + rollingUpdate: null + {{- end }} + {{- if not .Values.artifactBackend.autoscaling.enabled }} + replicas: {{ .Values.artifactBackend.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "core.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: artifact-backend + template: + metadata: + labels: + {{- include "core.matchLabels" . | nindent 8 }} + app.kubernetes.io/component: artifact-backend + annotations: + checksum/config: {{ include (print $.Template.BasePath "/artifact-backend/configmap.yaml") . | sha256sum }} + {{- with .Values.artifactBackend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + # Distroless nonroot:nonroot is 65532:65532 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + {{- if .Values.artifactBackend.serviceAccountName }} + serviceAccountName: {{ .Values.artifactBackend.serviceAccountName }} + {{- end }} + automountServiceAccountToken: {{ .Values.artifactBackend.automountServiceAccountToken | default false }} + terminationGracePeriodSeconds: 120 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + {{- if .Values.database.enabled }} + - name: wait-for-db + image: {{ .Values.database.image.repository }}:{{ .Values.database.image.tag }} + imagePullPolicy: {{ $.Values.database.image.pullPolicy }} + command: ['sh', '-c', "until pg_isready; do echo waiting for db; sleep 2; done"] + env: + - name: PGHOST + value: {{ template "core.database" . }} + - name: PGUSER + value: {{ template "core.database.username" . }} + {{- end }} + {{- if not .Values.artifactBackend.temporal.hostPort }} + - name: temporal-admin-tools + securityContext: + runAsUser: 0 + runAsGroup: 0 + image: {{ .Values.temporal.admintools.image.repository }}:{{ .Values.temporal.admintools.image.tag }} + imagePullPolicy: {{ .Values.temporal.admintools.image.pullPolicy }} + command: ["/bin/bash", "-c"] + args: + - > + until tctl cluster health 2>&1 > /dev/null; do echo waiting for Temporal; sleep 2; done && + if [[ ! $(tctl --namespace artifact-backend namespace list | grep artifact-backend) ]]; then tctl --namespace artifact-backend namespace register; fi + env: + - name: TEMPORAL_CLI_ADDRESS + value: "{{ template "core.temporal" . }}-frontend:{{ template "core.temporal.frontend.grpcPort" . }}" + {{- end }} + - name: artifact-backend-migration + image: {{ .Values.artifactBackend.image.repository }}:{{ .Values.artifactBackend.image.tag }} + imagePullPolicy: {{ .Values.artifactBackend.image.pullPolicy }} + {{- if .Values.artifactBackend.resources }} + resources: + {{- toYaml .Values.artifactBackend.resources | nindent 12 }} + {{- end }} + command: [./{{ .Values.artifactBackend.commandName.migration }}] + volumeMounts: + - name: config + mountPath: {{ .Values.artifactBackend.configPath }} + subPath: config.yaml + {{- with .Values.artifactBackend.extraInitContainers }} + {{- toYaml . | indent 8 }} + {{- end }} + containers: + - name: artifact-backend + image: {{ .Values.artifactBackend.image.repository }}:{{ .Values.artifactBackend.image.tag }} + imagePullPolicy: {{ .Values.artifactBackend.image.pullPolicy }} + readinessProbe: + httpGet: + path: /v1alpha/__readiness + scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }} + port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public + periodSeconds: 10 + livenessProbe: + httpGet: + path: /v1alpha/__liveness + scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }} + port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public + periodSeconds: 10 + {{- if .Values.artifactBackend.resources }} + resources: + {{- toYaml .Values.artifactBackend.resources | nindent 12 }} + {{- end }} + command: [./{{ .Values.artifactBackend.commandName.main }}] + ports: + - name: {{ ternary "https" "http" .Values.internalTLS.enabled }}-admin + containerPort: {{ template "core.artifactBackend.privatePort" . }} + protocol: TCP + - name: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public + containerPort: {{ template "core.artifactBackend.publicPort" . }} + protocol: TCP + volumeMounts: + - name: config + mountPath: {{ .Values.artifactBackend.configPath }} + subPath: config.yaml + {{- if .Values.internalTLS.enabled }} + - name: artifact-internal-certs + mountPath: "/etc/instill-ai/core/ssl/artifact" + {{- end }} + {{- with .Values.artifactBackend.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.artifactBackend.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: artifact-backend-worker + image: {{ .Values.artifactBackend.image.repository }}:{{ .Values.artifactBackend.image.tag }} + imagePullPolicy: {{ .Values.artifactBackend.image.pullPolicy }} + livenessProbe: + tcpSocket: + port: rpc + {{- if .Values.artifactBackend.resources }} + resources: + {{- toYaml .Values.artifactBackend.resources | nindent 12 }} + {{- end }} + command: [./{{ .Values.artifactBackend.commandName.worker }}] + volumeMounts: + - name: config + mountPath: {{ .Values.artifactBackend.configPath }} + subPath: config.yaml + {{- if .Values.internalTLS.enabled }} + - name: artifact-internal-certs + mountPath: "/etc/instill-ai/core/ssl/artifact" + {{- end }} + {{- with .Values.artifactBackend.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.artifactBackend.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.artifactBackend.sidecarContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "core.artifactBackend" . }} + {{- if .Values.internalTLS.enabled }} + - name: artifact-internal-certs + secret: + secretName: {{ template "core.internalTLS.artifactBackend.secretName" . }} + {{- end }} + {{- with .Values.artifactBackend.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.artifactBackend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.artifactBackend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.artifactBackend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/core/templates/artifact-backend/hpa.yml b/charts/core/templates/artifact-backend/hpa.yml new file mode 100644 index 00000000..ef53e311 --- /dev/null +++ b/charts/core/templates/artifact-backend/hpa.yml @@ -0,0 +1,33 @@ +{{- if .Values.artifactBackend.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "core.artifactBackend" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "core.artifactBackend" . }} + labels: + {{- include "core.labels" . | nindent 4 }} + app.kubernetes.io/component: artifact-backend + minReplicas: {{ .Values.artifactBackend.autoscaling.minReplicas }} + maxReplicas: {{ .Values.artifactBackend.autoscaling.maxReplicas }} + metrics: +{{- with .Values.artifactBackend.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} +{{- end }} +{{- with .Values.artifactBackend.autoscaling.targetAverageMemoryUtilization }} + - type: Resource + resource: + name: memory + target: + type: AverageValue + averageValue: {{ . }} +{{- end }} +{{- end }} diff --git a/charts/core/templates/artifact-backend/service.yaml b/charts/core/templates/artifact-backend/service.yaml new file mode 100644 index 00000000..66f19e1e --- /dev/null +++ b/charts/core/templates/artifact-backend/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "core.artifactBackend" . }} + labels: + {{- include "core.labels" . | nindent 4 }} + app.kubernetes.io/component: artifact-backend + {{- with .Values.artifactBackend.serviceAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - name: {{ ternary "https" "http" .Values.internalTLS.enabled }}-private + port: {{ template "core.artifactBackend.privatePort" . }} + targetPort: {{ template "core.artifactBackend.privatePort" . }} + - name: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public + port: {{ template "core.artifactBackend.publicPort" . }} + targetPort: {{ template "core.artifactBackend.publicPort" . }} + selector: + {{- include "core.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: artifact-backend diff --git a/charts/core/values.yaml b/charts/core/values.yaml index a1f9d03d..a2c5da95 100644 --- a/charts/core/values.yaml +++ b/charts/core/values.yaml @@ -623,6 +623,76 @@ controllerModel: spec: minAvailable: maxUnavailable: +# -- The configuration of artifact-backend +artifactBackend: + # -- The image of artifact-backend + image: + repository: instill/artifact-backend + tag: 0.2.1-alpha + pullPolicy: IfNotPresent + # -- The command names to be executed + commandName: + migration: artifact-backend-migrate + main: artifact-backend + worker: artifact-backend-worker + # -- The path of configuration file for artifact-backend + configPath: /artifact-backend/config/config.yaml + # -- The database migration version + dbVersion: 1 + # -- Set the service account to be used, default if left empty + serviceAccountName: "" + # -- Mount the service account token + automountServiceAccountToken: false + # -- The number of replica for artifact-backend + replicaCount: 1 + # -- Add extra env variables + extraEnv: [] + # -- Mount external volumes + # For example, mount a secret containing Certificate root CA to verify database + # TLS connection. + extraVolumes: [] + # - name: my-volume + # secret: + # secretName: my-secret + extraVolumeMounts: [] + # - name: my-volume + # mountPath: /etc/secrets/my-secret + # readOnly: true + # -- Add extra init containers + extraInitContainers: [] + # extraInitContainers: + # - name: ... + # image: ... + # -- Add extra sidecar containers + sidecarContainers: {} + # -- Additional deployment annotations + podAnnotations: {} + # -- Additional service annotations + serviceAnnotations: {} + # -- The pod resource + resources: {} + nodeSelector: {} + tolerations: [] + affinity: {} + autoscaling: + enabled: false + minReplicas: + maxReplicas: + targetCPUUtilizationPercentage: + targetAverageMemoryUtilization: + podDisruptionBudget: + enabled: false + spec: + minAvailable: + maxUnavailable: + # -- The configuration of Temporal Cloud + temporal: + hostPort: + namespace: + ca: + cert: + key: + serverName: # -- The configuration of console console: # -- Enable console deployment or not diff --git a/docker-compose-build.yml b/docker-compose-build.yml index c7b381e9..697506e1 100644 --- a/docker-compose-build.yml +++ b/docker-compose-build.yml @@ -9,6 +9,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${API_GATEWAY_IMAGE}:${API_GATEWAY_VERSION} build: context: ./${API_GATEWAY_HOST} @@ -26,6 +27,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${MGMT_BACKEND_IMAGE}:${MGMT_BACKEND_VERSION} build: context: ./${MGMT_BACKEND_HOST} @@ -42,6 +44,7 @@ services: - exclude-console - exclude-model - exclude-controller-model + - exclude-artifact image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION} build: context: ./${PIPELINE_BACKEND_HOST} @@ -58,6 +61,7 @@ services: - exclude-console - exclude-pipeline - exclude-controller-model + - exclude-artifact image: ${MODEL_BACKEND_IMAGE}:${MODEL_BACKEND_VERSION} build: context: ./${MODEL_BACKEND_HOST} @@ -76,6 +80,7 @@ services: - exclude-console - exclude-pipeline - exclude-model + - exclude-artifact image: ${CONTROLLER_MODEL_IMAGE}:${CONTROLLER_MODEL_VERSION} build: context: ./${CONTROLLER_MODEL_HOST} @@ -92,6 +97,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${CONSOLE_IMAGE}:${CONSOLE_VERSION} build: context: ./${CONSOLE_HOST} @@ -104,9 +110,26 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${CONSOLE_IMAGE}-playwright:${CONSOLE_VERSION} build: context: ./${CONSOLE_HOST} dockerfile: Dockerfile.playwright args: TEST_USER: "root" + artifact_backend: + profiles: + - all + - exclude-api-gateway + - exclude-mgmt + - exclude-console + - exclude-pipeline + - exclude-model + - exclude-controller-model + image: ${ARTIFACT_BACKEND_IMAGE}:${ARTIFACT_BACKEND_VERSION} + build: + context: ./${ARTIFACT_BACKEND_HOST} + args: + SERVICE_NAME: ${ARTIFACT_BACKEND_HOST} + GOLANG_VERSION: ${GOLANG_VERSION} + K6_VERSION: ${K6_VERSION} diff --git a/docker-compose-latest.yml b/docker-compose-latest.yml index 4cfae10d..96d059ad 100644 --- a/docker-compose-latest.yml +++ b/docker-compose-latest.yml @@ -9,6 +9,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${API_GATEWAY_IMAGE}:latest environment: LOG_LEVEL: DEBUG @@ -21,6 +22,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${MGMT_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -37,6 +39,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${MGMT_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -50,6 +53,7 @@ services: - exclude-console - exclude-model - exclude-controller-model + - exclude-artifact image: ${PIPELINE_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -66,6 +70,7 @@ services: - exclude-console - exclude-model - exclude-controller-model + - exclude-artifact image: ${PIPELINE_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -79,6 +84,7 @@ services: - exclude-console - exclude-pipeline - exclude-controller-model + - exclude-artifact image: ${MODEL_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -91,6 +97,7 @@ services: - exclude-console - exclude-pipeline - exclude-controller-model + - exclude-artifact image: ${MODEL_BACKEND_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -107,11 +114,29 @@ services: - exclude-console - exclude-pipeline - exclude-controller-model + - exclude-artifact image: ${MODEL_BACKEND_IMAGE}:latest environment: CFG_INITMODEL_ENABLED: ${INITMODEL_ENABLED} CFG_INITMODEL_PATH: https://raw.githubusercontent.com/instill-ai/model/main/model-hub/model_hub_cpu.json + artifact_backend: + profiles: + - all + - exclude-api-gateway + - exclude-mgmt + - exclude-console + - exclude-pipeline + - exclude-model + - exclude-controller-model + image: ${ARTIFACT_BACKEND_IMAGE}:latest + environment: + CFG_SERVER_DEBUG: "true" + CFG_SERVER_EDITION: ${EDITION} + ports: + - ${ARTIFACT_BACKEND_PRIVATEPORT}:${ARTIFACT_BACKEND_PRIVATEPORT} + - ${ARTIFACT_BACKEND_PUBLICPORT}:${ARTIFACT_BACKEND_PUBLICPORT} + controller_model: profiles: - all @@ -120,6 +145,7 @@ services: - exclude-console - exclude-pipeline - exclude-model + - exclude-artifact image: ${CONTROLLER_MODEL_IMAGE}:latest environment: CFG_SERVER_DEBUG: "true" @@ -135,6 +161,7 @@ services: - exclude-pipeline - exclude-model - exclude-controller-model + - exclude-artifact image: ${CONSOLE_IMAGE}:latest environment: NEXT_PUBLIC_USAGE_COLLECTION_ENABLED: ${USAGE_ENABLED} diff --git a/docker-compose.yml b/docker-compose.yml index d39fbf95..542aff77 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,10 @@ services: PIPELINE_BACKEND_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT} MODEL_BACKEND_HOST: ${MODEL_BACKEND_HOST} MODEL_BACKEND_PUBLICPORT: ${MODEL_BACKEND_PUBLICPORT} + MODEL_BACKEND_PRIVATEPORT: ${MODEL_BACKEND_PRIVATEPORT} + ARTIFACT_BACKEND_HOST: ${ARTIFACT_BACKEND_HOST} + ARTIFACT_BACKEND_PUBLICPORT: ${ARTIFACT_BACKEND_PUBLICPORT} + ARTIFACT_BACKEND_PRIVATEPORT: ${ARTIFACT_BACKEND_PRIVATEPORT} INFLUXDB_HOST: ${INFLUXDB_HOST} INFLUXDB_PORT: ${INFLUXDB_PORT} JAEGER_ENABLED: ${OBSERVE_ENABLED} @@ -351,6 +355,45 @@ services: volumes: - model_repository:/model-repository + artifact_backend: + container_name: ${ARTIFACT_BACKEND_HOST} + image: ${ARTIFACT_BACKEND_IMAGE}:${ARTIFACT_BACKEND_VERSION} + restart: unless-stopped + environment: + CFG_SERVER_PRIVATEPORT: ${ARTIFACT_BACKEND_PRIVATEPORT} + CFG_SERVER_PUBLICPORT: ${ARTIFACT_BACKEND_PUBLICPORT} + CFG_SERVER_DEBUG: "false" + CFG_SERVER_USAGE_ENABLED: ${USAGE_ENABLED} + CFG_SERVER_EDITION: ${EDITION} + CFG_REGISTRY_HOST: ${REGISTRY_HOST} + CFG_REGISTRY_PORT: ${REGISTRY_PORT} + CFG_DATABASE_HOST: ${POSTGRESQL_HOST} + CFG_DATABASE_PORT: ${POSTGRESQL_PORT} + CFG_DATABASE_USERNAME: postgres + CFG_DATABASE_PASSWORD: password + CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED} + CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT} + command: + - /bin/sh + - -c + - | + ./artifact-backend-migrate + ./artifact-backend + healthcheck: + test: + [ + "CMD-SHELL", + "wget --no-verbose --tries=1 --spider http://${ARTIFACT_BACKEND_HOST}:${ARTIFACT_BACKEND_PUBLICPORT}/v1alpha/health/artifact", + ] + start_period: 20s + start_interval: 1s + interval: 1s + timeout: 5s + retries: 10 + depends_on: + pg_sql: + condition: service_healthy + console: container_name: ${CONSOLE_HOST} image: ${CONSOLE_IMAGE}:${CONSOLE_VERSION}