diff --git a/.env b/.env index 65bfc81b..c24baf86 100644 --- a/.env +++ b/.env @@ -1,8 +1,8 @@ # docker compose project name COMPOSE_PROJECT_NAME=instill-vdp -# system-wise config path (all base, vdp, and model projects must use the same path) -SYSTEM_CONFIG_PATH=~/.config/instill +# system-wise config path (all core, vdp, and model projects must use the same path) +SYSTEM_CONFIG_PATH=$(HOME)/.config/instill # configuration directory path for docker build BUILD_CONFIG_DIR_PATH=. @@ -26,13 +26,13 @@ COMPOSE_DOCKER_CLI_BUILD=1 # version ALPINE_VERSION=3.16 -GOLANG_VERSION=1.19.3 +GOLANG_VERSION=1.21 KRAKEND_CE_VERSION=2.1.3 ARTIVC_VERSION=0.10.0 K6_VERSION=0.44.0 -# Instill Base -INSTILL_BASE_VERSION=0.4.1-alpha +# Instill Core +INSTILL_CORE_VERSION=0.4.1-alpha # api-gateway API_GATEWAY_HOST=api-gateway @@ -53,10 +53,6 @@ CONNECTOR_BACKEND_HOST=connector-backend CONNECTOR_BACKEND_PRIVATEPORT=3082 CONNECTOR_BACKEND_PUBLICPORT=8082 -# model-backend -MODEL_BACKEND_HOST=model-backend -MODEL_BACKEND_PUBLICPORT=8083 - # mgmt-backend MGMT_BACKEND_HOST=mgmt-backend MGMT_BACKEND_PRIVATEPORT=3084 @@ -87,17 +83,8 @@ REDIS_PORT=6379 ETCD_HOST=etcd ETCD_CLIENT_PORT=3379 -# influxdb -INFLUXDB_HOST=influxdb -INFLUXDB_PORT=8086 - # otel OTEL_COLLECTOR_PORT=8095 -OTEL_COLLECTOR_PROMETHEUS_PORT=9001 - -# jaeger -JAEGER_HOST=jaeger -JAEGER_LISTEN_THRIFT_PORT=14268 # accept jaeger.thrift directly from clients # Temopral TEMPORAL_ADMIN_TOOLS_IMAGE=temporalio/admin-tools diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 63e53730..e6953ac8 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -31,7 +31,7 @@ $ make latest PROFILE=all The env variable `PROFILE` is intended to specify which service component you want to develop on - `all` - When you set `PROFILE=all`, the whole `Instill Base` and `Instill VDP` stack will be launched, meaning you want to test the system as a whole + When you set `PROFILE=all`, the whole `Instill Core` and `Instill VDP` stack will be launched, meaning you want to test the system as a whole - `{service}` diff --git a/.github/workflows/helm-integration-test-backend.yml b/.github/workflows/helm-integration-test-backend.yml index 15cc6e0b..dfc6fd8f 100644 --- a/.github/workflows/helm-integration-test-backend.yml +++ b/.github/workflows/helm-integration-test-backend.yml @@ -46,35 +46,35 @@ jobs: run: | curl https://github.com/grafana/k6/releases/download/v${{ env.K6_VERSION }}/k6-v${{ env.K6_VERSION }}-linux-amd64.tar.gz -L | tar xvz --strip-components 1 && sudo cp k6 /usr/bin - - name: Checkout repo (base) + - name: Checkout repo (core) uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Load .env file (base) + - name: Load .env file (core) uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Launch Helm Instill Base (latest) + - name: Launch Helm Instill Core (latest) run: | - helm install base charts/base --namespace instill-ai --create-namespace \ + helm install core charts/core --namespace instill-ai --create-namespace \ --set edition=k8s-ce:test \ --set apiGateway.image.tag=latest \ --set mgmtBackend.image.tag=latest \ --set console.image.tag=latest \ --set tags.observability=false - - name: Wait for base pods up + - name: Wait for core pods up run: | - while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do + while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do echo "$(kubectl get pods --namespace instill-ai)" sleep 10 done - name: Port-forward of api-gateway run: | - API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name') + API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o json | jq -r '.items[0].metadata.name') kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @@ -110,7 +110,7 @@ jobs: helm-integration-test-latest-mac: if: inputs.target == 'latest' && github.ref == 'refs/heads/main' - runs-on: [self-hosted,macOS,vdp] + runs-on: [self-hosted, macOS, vdp] steps: - name: Set up environment run: | @@ -148,48 +148,48 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - name: Load .env file uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Check if Base Helm release exists - id: check-base-helm-release + - name: Check if Core Helm release exists + id: check-core-helm-release run: | - if helm ls -n instill-ai | grep -q 'base'; then - echo "Helm release 'base' found." + if helm ls -n instill-ai | grep -q 'core'; then + echo "Helm release 'core' found." echo "release_exists=true" >> $GITHUB_OUTPUT else - echo "Helm release 'base' not found." + echo "Helm release 'core' not found." fi - - name: Uninstall Base Helm Release - if: steps.check-base-helm-release.outputs.release_exists == 'true' + - name: Uninstall Core Helm Release + if: steps.check-core-helm-release.outputs.release_exists == 'true' run: | - helm uninstall base --namespace instill-ai + helm uninstall core --namespace instill-ai kubectl delete namespace instill-ai - - name: Launch Helm Instill Base (latest) + - name: Launch Helm Instill Core (latest) run: | - helm install base charts/base --namespace instill-ai --create-namespace \ + helm install core charts/core --namespace instill-ai --create-namespace \ --set edition=k8s-ce:test \ --set apiGateway.image.tag=latest \ --set mgmtBackend.image.tag=latest \ --set console.image.tag=latest \ --set tags.observability=false - - name: Wait for base pods up + - name: Wait for core pods up run: | - while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do + while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do echo "$(kubectl get pods --namespace instill-ai)" sleep 10 done - name: Port-forward of api-gateway run: | - API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name') + API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o json | jq -r '.items[0].metadata.name') kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @@ -224,11 +224,11 @@ jobs: cd ${{ inputs.component }} make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT} - - name: Uninstall Base and VDP Helm Release + - name: Uninstall Core and VDP Helm Release run: | helm uninstall vdp --namespace instill-ai - helm uninstall base --namespace instill-ai - kubectl delete namespace instill-ai + helm uninstall core --namespace instill-ai + kubectl delete namespace instill-ai helm-integration-test-release-linux: if: inputs.target == 'release' @@ -265,35 +265,35 @@ jobs: run: | curl https://github.com/grafana/k6/releases/download/v${{ env.K6_VERSION }}/k6-v${{ env.K6_VERSION }}-linux-amd64.tar.gz -L | tar xvz --strip-components 1 && sudo cp k6 /usr/bin - - name: Checkout repo (base) + - name: Checkout repo (core) uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Load .env file (base) + - name: Load .env file (core) uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Launch Helm Instill Base (release) + - name: Launch Helm Instill Core (release) run: | - helm install base charts/base --namespace instill-ai --create-namespace \ + helm install core charts/core --namespace instill-ai --create-namespace \ --set edition=k8s-ce:test \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set tags.observability=false \ - --set tags.prometheusStack=false + --set tags.prometheusStack=false - - name: Wait for base pods up + - name: Wait for core pods up run: | - while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do + while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do echo "$(kubectl get pods --namespace instill-ai)" sleep 10 done - name: Port-forward of api-gateway run: | - API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name') + API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o json | jq -r '.items[0].metadata.name') kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @@ -335,7 +335,7 @@ jobs: helm-integration-test-release-mac: if: inputs.target == 'release' - runs-on: [self-hosted,macOS,vdp] + runs-on: [self-hosted, macOS, vdp] steps: - name: Set up environment run: | @@ -373,48 +373,48 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - name: Load .env file uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Check if Base Helm release exists - id: check-base-helm-release + - name: Check if Core Helm release exists + id: check-core-helm-release run: | - if helm ls -n instill-ai | grep -q 'base'; then - echo "Helm release 'base' found." + if helm ls -n instill-ai | grep -q 'core'; then + echo "Helm release 'core' found." echo "release_exists=true" >> $GITHUB_OUTPUT else - echo "Helm release 'base' not found." + echo "Helm release 'core' not found." fi - - name: Uninstall Base Helm Release - if: steps.check-base-helm-release.outputs.release_exists == 'true' + - name: Uninstall Core Helm Release + if: steps.check-core-helm-release.outputs.release_exists == 'true' run: | - helm uninstall base --namespace instill-ai + helm uninstall core --namespace instill-ai kubectl delete namespace instill-ai - - name: Launch Helm Instill Base (release) + - name: Launch Helm Instill Core (release) run: | - helm install base charts/base --namespace instill-ai --create-namespace \ + helm install core charts/core --namespace instill-ai --create-namespace \ --set edition=k8s-ce:test \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ --set tags.observability=false \ --set tags.prometheusStack=false - - name: Wait for base pods up + - name: Wait for core pods up run: | - while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do + while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do echo "$(kubectl get pods --namespace instill-ai)" sleep 10 done - - name: Port-forward of base-api-gateway + - name: Port-forward of core-api-gateway run: | - API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name') + API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o json | jq -r '.items[0].metadata.name') kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @@ -455,8 +455,8 @@ jobs: cd ${{ inputs.component }} make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT} - - name: Uninstall Base and VDP Helm Release + - name: Uninstall Core and VDP Helm Release run: | helm uninstall vdp --namespace instill-ai - helm uninstall base --namespace instill-ai + helm uninstall core --namespace instill-ai kubectl delete namespace instill-ai diff --git a/.github/workflows/integration-test-backend.yml b/.github/workflows/integration-test-backend.yml index 45bf94d2..f9e69dc1 100644 --- a/.github/workflows/integration-test-backend.yml +++ b/.github/workflows/integration-test-backend.yml @@ -43,17 +43,17 @@ jobs: rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - - name: Checkout repo (base) + - name: Checkout repo (core) uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Load .env file (base) + - name: Load .env file (core) uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Launch Instill Base (latest) + - name: Launch Instill Core (latest) run: | COMPOSE_PROFILES=all \ EDITION=local-ce:test \ @@ -90,7 +90,7 @@ jobs: integration-test-latest-mac: if: inputs.target == 'latest' && github.ref == 'refs/heads/main' - runs-on: [self-hosted,macOS,vdp] + runs-on: [self-hosted, macOS, vdp] steps: - name: Set up environment run: | @@ -122,25 +122,25 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - name: Load .env file uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Make down base + - name: Make down core run: | - docker rm -f base-build-latest >/dev/null 2>&1 - docker rm -f base-backend-integration-test-latest >/dev/null 2>&1 - docker rm -f base-console-integration-test-latest >/dev/null 2>&1 - docker rm -f base-backend-integration-test-helm-latest >/dev/null 2>&1 - docker rm -f base-console-integration-test-helm-latest >/dev/null 2>&1 - docker rm -f base-dind-latest >/dev/null 2>&1 + docker rm -f core-build-latest >/dev/null 2>&1 + docker rm -f core-backend-integration-test-latest >/dev/null 2>&1 + docker rm -f core-console-integration-test-latest >/dev/null 2>&1 + docker rm -f core-backend-integration-test-helm-latest >/dev/null 2>&1 + docker rm -f core-console-integration-test-helm-latest >/dev/null 2>&1 + docker rm -f core-dind-latest >/dev/null 2>&1 EDITION=NULL docker compose -f docker-compose.yml -f docker-compose.observe.yml down -v sleep 60 - - name: Launch Instill Base (latest) + - name: Launch Instill Core (latest) run: | make build-latest COMPOSE_PROFILES=all \ @@ -188,18 +188,18 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Make down base + - name: Make down core run: | - docker rm -f base-build-latest >/dev/null 2>&1 - docker rm -f base-backend-integration-test-latest >/dev/null 2>&1 - docker rm -f base-console-integration-test-latest >/dev/null 2>&1 - docker rm -f base-backend-integration-test-helm-latest >/dev/null 2>&1 - docker rm -f base-console-integration-test-helm-latest >/dev/null 2>&1 - docker rm -f base-dind-latest >/dev/null 2>&1 + docker rm -f core-build-latest >/dev/null 2>&1 + docker rm -f core-backend-integration-test-latest >/dev/null 2>&1 + docker rm -f core-console-integration-test-latest >/dev/null 2>&1 + docker rm -f core-backend-integration-test-helm-latest >/dev/null 2>&1 + docker rm -f core-console-integration-test-helm-latest >/dev/null 2>&1 + docker rm -f core-dind-latest >/dev/null 2>&1 EDITION=NULL docker compose -f docker-compose.yml -f docker-compose.observe.yml down -v - sleep 60 + sleep 60 integration-test-release-linux: if: inputs.target == 'release' @@ -233,17 +233,17 @@ jobs: rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - - name: Checkout repo (base) + - name: Checkout repo (core) uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Load .env file (base) + - name: Load .env file (core) uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Launch Instill Base (latest) + - name: Launch Instill Core (latest) run: | EDITION=local-ce:test \ docker compose up -d --quiet-pull @@ -282,7 +282,7 @@ jobs: integration-test-release-mac: if: inputs.target == 'release' - runs-on: [self-hosted,macOS,vdp] + runs-on: [self-hosted, macOS, vdp] steps: - name: Set up environment run: | @@ -314,25 +314,25 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - name: Load .env file uses: cardinalby/export-env-action@v2 with: envFile: .env - - name: Make down base + - name: Make down core run: | - docker rm -f base-build-release >/dev/null 2>&1 - docker rm -f base-backend-integration-test-release >/dev/null 2>&1 - docker rm -f base-console-integration-test-release >/dev/null 2>&1 - docker rm -f base-backend-integration-test-helm-release >/dev/null 2>&1 - docker rm -f base-console-integration-test-helm-release >/dev/null 2>&1 - docker rm -f base-dind-release >/dev/null 2>&1 + docker rm -f core-build-release >/dev/null 2>&1 + docker rm -f core-backend-integration-test-release >/dev/null 2>&1 + docker rm -f core-console-integration-test-release >/dev/null 2>&1 + docker rm -f core-backend-integration-test-helm-release >/dev/null 2>&1 + docker rm -f core-console-integration-test-helm-release >/dev/null 2>&1 + docker rm -f core-dind-release >/dev/null 2>&1 EDITION=NULL docker compose -f docker-compose.yml -f docker-compose.observe.yml down -v sleep 60 - - name: Launch Instill Base (release) + - name: Launch Instill Core (release) run: | make build-release EDITION=local-ce:test \ @@ -382,15 +382,15 @@ jobs: - name: Checkout repo uses: actions/checkout@v3 with: - repository: instill-ai/base + repository: instill-ai/core - - name: Make down base + - name: Make down core run: | - docker rm -f base-build-release >/dev/null 2>&1 - docker rm -f base-backend-integration-test-release >/dev/null 2>&1 - docker rm -f base-console-integration-test-release >/dev/null 2>&1 - docker rm -f base-backend-integration-test-helm-release >/dev/null 2>&1 - docker rm -f base-console-integration-test-helm-release >/dev/null 2>&1 - docker rm -f base-dind-release >/dev/null 2>&1 + docker rm -f core-build-release >/dev/null 2>&1 + docker rm -f core-backend-integration-test-release >/dev/null 2>&1 + docker rm -f core-console-integration-test-release >/dev/null 2>&1 + docker rm -f core-backend-integration-test-helm-release >/dev/null 2>&1 + docker rm -f core-console-integration-test-helm-release >/dev/null 2>&1 + docker rm -f core-dind-release >/dev/null 2>&1 EDITION=NULL docker compose -f docker-compose.yml -f docker-compose.observe.yml down -v sleep 60 diff --git a/.github/workflows/make-all.yml b/.github/workflows/make-all.yml index cd34203f..fb6218c4 100644 --- a/.github/workflows/make-all.yml +++ b/.github/workflows/make-all.yml @@ -61,7 +61,7 @@ jobs: - name: Probe to vdp services healthcheck endpoint run: | - curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/base/v1alpha/health/mgmt + curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/core/v1alpha/health/mgmt curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector diff --git a/.github/workflows/make-latest.yml b/.github/workflows/make-latest.yml index 74a8f9b2..c0902ea8 100644 --- a/.github/workflows/make-latest.yml +++ b/.github/workflows/make-latest.yml @@ -60,7 +60,7 @@ jobs: - name: Curl to vdp services healthcheck endpoint run: | - curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/base/v1alpha/health/mgmt + curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/core/v1alpha/health/mgmt curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3085/v1alpha/health/controller diff --git a/Dockerfile b/Dockerfile index ae33b88b..f44017e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN echo "VDP latest codebase cloned on ${CACHE_DATE}" WORKDIR /instill-ai -RUN git clone https://github.com/instill-ai/base.git +RUN git clone https://github.com/instill-ai/core.git WORKDIR /instill-ai/vdp @@ -48,8 +48,8 @@ RUN echo "VDP release codebase cloned on ${CACHE_DATE}" WORKDIR /instill-ai -ARG INSTILL_BASE_VERSION -RUN git clone -b v${INSTILL_BASE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/base.git +ARG INSTILL_CORE_VERSION +RUN git clone -b v${INSTILL_CORE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/core.git WORKDIR /instill-ai/vdp diff --git a/Makefile b/Makefile index 444a262a..bd71aa93 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ HELM_RELEASE_NAME := vdp .PHONY: all all: ## Launch all services with their up-to-date release version @make build-release - @if ! (docker compose ls -q | grep -q "instill-base"); then \ + @if ! (docker compose ls -q | grep -q "instill-core"); then \ export TMP_CONFIG_DIR=$(shell mktemp -d) && \ export SYSTEM_CONFIG_PATH=$(shell eval echo ${SYSTEM_CONFIG_PATH}) && \ docker run --rm \ @@ -30,10 +30,10 @@ all: ## Launch all services with their up-to-date release version -v $${SYSTEM_CONFIG_PATH}:$${SYSTEM_CONFIG_PATH} \ --name ${CONTAINER_COMPOSE_NAME}-release \ ${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \ - cp /instill-ai/base/.env $${TMP_CONFIG_DIR}/.env && \ - cp /instill-ai/base/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ - cp -r /instill-ai/base/configs/influxdb $${TMP_CONFIG_DIR} && \ - /bin/sh -c 'cd /instill-ai/base && make all EDITION=$${EDITION:=local-ce} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \ + cp /instill-ai/core/.env $${TMP_CONFIG_DIR}/.env && \ + cp /instill-ai/core/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ + cp -r /instill-ai/core/configs/influxdb $${TMP_CONFIG_DIR} && \ + /bin/sh -c 'cd /instill-ai/core && make all EDITION=$${EDITION:=local-ce} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \ rm -rf $${TMP_CONFIG_DIR}/* \ " && \ rm -rf $${TMP_CONFIG_DIR}; \ @@ -43,7 +43,7 @@ all: ## Launch all services with their up-to-date release version .PHONY: latest latest: ## Lunch all dependent services with their latest codebase @make build-latest - @if ! (docker compose ls -q | grep -q "instill-base"); then \ + @if ! (docker compose ls -q | grep -q "instill-core"); then \ export TMP_CONFIG_DIR=$(shell mktemp -d) && \ export SYSTEM_CONFIG_PATH=$(shell eval echo ${SYSTEM_CONFIG_PATH}) && \ docker run --rm \ @@ -52,10 +52,10 @@ latest: ## Lunch all dependent services with their latest codebase -v $${SYSTEM_CONFIG_PATH}:$${SYSTEM_CONFIG_PATH} \ --name ${CONTAINER_COMPOSE_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - cp /instill-ai/base/.env $${TMP_CONFIG_DIR}/.env && \ - cp /instill-ai/base/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ - cp -r /instill-ai/base/configs/influxdb $${TMP_CONFIG_DIR} && \ - /bin/sh -c 'cd /instill-ai/base && make latest PROFILE=all EDITION=$${EDITION:=local-ce:latest} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}'&& \ + cp /instill-ai/core/.env $${TMP_CONFIG_DIR}/.env && \ + cp /instill-ai/core/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ + cp -r /instill-ai/core/configs/influxdb $${TMP_CONFIG_DIR} && \ + /bin/sh -c 'cd /instill-ai/core && make latest PROFILE=all EDITION=$${EDITION:=local-ce:latest} BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR} SYSTEM_CONFIG_PATH=$${SYSTEM_CONFIG_PATH} OBSERVE_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}'&& \ rm -rf $${TMP_CONFIG_DIR}/* \ " && \ rm -rf $${TMP_CONFIG_DIR}; \ @@ -77,7 +77,7 @@ stop: ## Stop all components -v /var/run/docker.sock:/var/run/docker.sock \ --name ${CONTAINER_COMPOSE_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && make stop' \ + /bin/sh -c 'cd /instill-ai/core && make stop' \ " .PHONY: start @@ -86,7 +86,7 @@ start: ## Start all stopped components -v /var/run/docker.sock:/var/run/docker.sock \ --name ${CONTAINER_COMPOSE_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && make start' \ + /bin/sh -c 'cd /instill-ai/core && make start' \ " @EDITION= docker compose start @@ -101,20 +101,20 @@ down: ## Stop all services and remove all service containers and volumes @docker rm -f ${CONTAINER_COMPOSE_NAME}-latest >/dev/null 2>&1 @docker rm -f ${CONTAINER_COMPOSE_NAME}-release >/dev/null 2>&1 @EDITION= docker compose down -v - @if docker compose ls -q | grep -q "instill-base"; then \ + @if docker compose ls -q | grep -q "instill-core"; then \ if docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:latest >/dev/null 2>&1; then \ docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ --name ${CONTAINER_COMPOSE_NAME} \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && make down' \ + /bin/sh -c 'cd /instill-ai/core && make down' \ "; \ elif docker image inspect ${CONTAINER_COMPOSE_IMAGE_NAME}:release >/dev/null 2>&1; then \ docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ --name ${CONTAINER_COMPOSE_NAME} \ ${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && make down' \ + /bin/sh -c 'cd /instill-ai/core && make down' \ "; \ fi \ fi @@ -159,7 +159,7 @@ build-release: ## Build release images for all VDP components --build-arg GOLANG_VERSION=${GOLANG_VERSION} \ --build-arg K6_VERSION=${K6_VERSION} \ --build-arg CACHE_DATE="$(shell date)" \ - --build-arg INSTILL_BASE_VERSION=${INSTILL_BASE_VERSION} \ + --build-arg INSTILL_CORE_VERSION=${INSTILL_CORE_VERSION} \ --build-arg PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \ --build-arg CONNECTOR_BACKEND_VERSION=${CONNECTOR_BACKEND_VERSION} \ --build-arg CONTROLLER_VDP_VERSION=${CONTROLLER_VDP_VERSION} \ @@ -171,7 +171,7 @@ build-release: ## Build release images for all VDP components -v ${BUILD_CONFIG_DIR_PATH}/docker-compose.build.yml:/instill-ai/vdp/docker-compose.build.yml \ --name ${CONTAINER_BUILD_NAME}-release \ ${CONTAINER_COMPOSE_IMAGE_NAME}:release /bin/sh -c " \ - INSTILL_BASE_VERSION=${INSTILL_BASE_VERSION} \ + INSTILL_CORE_VERSION=${INSTILL_CORE_VERSION} \ PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \ CONNECTOR_BACKEND_VERSION=${CONNECTOR_BACKEND_VERSION} \ CONTROLLER_VDP_VERSION=${CONTROLLER_VDP_VERSION} \ @@ -214,11 +214,11 @@ helm-integration-test-latest: ## Run integration test on t ${DOCKER_HELM_IT_EXTRA_PARAMS} \ --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - cp /instill-ai/base/.env $${TMP_CONFIG_DIR}/.env && \ - cp /instill-ai/base/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ - /bin/sh -c 'cd /instill-ai/base && make build-latest BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \ - /bin/sh -c 'cd /instill-ai/base && \ - helm install base charts/base \ + cp /instill-ai/core/.env $${TMP_CONFIG_DIR}/.env && \ + cp /instill-ai/core/docker-compose.build.yml $${TMP_CONFIG_DIR}/docker-compose.build.yml && \ + /bin/sh -c 'cd /instill-ai/core && make build-latest BUILD_CONFIG_DIR_PATH=$${TMP_CONFIG_DIR}' && \ + /bin/sh -c 'cd /instill-ai/core && \ + helm install core charts/core \ --namespace ${HELM_NAMESPACE} --create-namespace \ --set edition=k8s-ce:test \ --set apiGateway.image.tag=latest \ @@ -228,8 +228,8 @@ helm-integration-test-latest: ## Run integration test on t --set tags.prometheusStack=false' \ /bin/sh -c 'rm -rf $${TMP_CONFIG_DIR}/*' \ " && rm -rf $${TMP_CONFIG_DIR} - @kubectl rollout status deployment base-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s - @export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o jsonpath="{.items[0].metadata.name}") && \ + @kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s + @export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o jsonpath="{.items[0].metadata.name}") && \ kubectl --namespace ${HELM_NAMESPACE} port-forward $${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & @while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @helm install ${HELM_RELEASE_NAME} charts/vdp --namespace ${HELM_NAMESPACE} --create-namespace \ @@ -261,7 +261,7 @@ endif ${DOCKER_HELM_IT_EXTRA_PARAMS} \ --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && helm uninstall base --namespace ${HELM_NAMESPACE}' \ + /bin/sh -c 'cd /instill-ai/core && helm uninstall core --namespace ${HELM_NAMESPACE}' \ " @kubectl delete namespace instill-ai @pkill -f "port-forward" @@ -275,9 +275,9 @@ helm-integration-test-release: ## Run integration test on ${DOCKER_HELM_IT_EXTRA_PARAMS} \ --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && \ + /bin/sh -c 'cd /instill-ai/core && \ export $(grep -v '^#' .env | xargs) && \ - helm install base charts/base \ + helm install core charts/core \ --namespace ${HELM_NAMESPACE} --create-namespace \ --set apiGateway.image.tag=${API_GATEWAY_VERSION} \ --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \ @@ -285,8 +285,8 @@ helm-integration-test-release: ## Run integration test on --set tags.observability=false \ --set tags.prometheusStack=false' \ " - @kubectl rollout status deployment base-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s - @export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o jsonpath="{.items[0].metadata.name}") && \ + @kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s + @export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o jsonpath="{.items[0].metadata.name}") && \ kubectl --namespace ${HELM_NAMESPACE} port-forward $${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 & @while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done @helm install ${HELM_RELEASE_NAME} charts/vdp --namespace ${HELM_NAMESPACE} --create-namespace \ @@ -318,7 +318,7 @@ endif ${DOCKER_HELM_IT_EXTRA_PARAMS} \ --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \ ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \ - /bin/sh -c 'cd /instill-ai/base && helm uninstall base --namespace ${HELM_NAMESPACE}' \ + /bin/sh -c 'cd /instill-ai/core && helm uninstall core --namespace ${HELM_NAMESPACE}' \ " @kubectl delete namespace instill-ai @pkill -f "port-forward" diff --git a/README.md b/README.md index 94bc06d7..74edead4 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,37 @@ Explore our [documentation](https://www.instill.tech/docs/vdp/deployment/overvie ## Dive into the Projects -Instill VDP stands as a crucial project within our **[Instill Core](https://github.com/instill-ai#instill-core)** stack. It provides an open-source AI infrastructure tailored for unstructured data, enabling versatile AI application development. Within this ecosystem, delve into source-available projects that enable you to construct flexible AI pipelines, enhancing your data processing abilities and converting raw data into actionable insights. +Explore our open-source unstructured data infrastructure stack, comprising a collection of source-available projects designed to streamline every aspect of building versatile AI features with unstructured data. Dive into the potential in our [documentation](https://www.instill.tech/docs). -- 💧 [Instill VDP](https://github.com/instill-ai/vdp): Streamline the unstructured data journey for AI applications -- ⚗️ [Instill Model](https://github.com/instill-ai/model): Transform your applications with AI models -- 🗿 [Instill Base](https://github.com/instill-ai/base): Essential Services for VDP and Model +
+ +
+ Open Source Unstructured Data Infrastructure Stack +
+
+
+ 🔮 Instill Core: The starting point for self-hosting Instill VDP and Instill Model
+ + **Instill Core**, or **Core**, serves as the bedrock upon which open-source unstructured data stack thrive. Essential services such as user management servers, databases, and third-party observability tools find their home here. Instill Core also provides deployment codes to facilitate the seamless launch of both Instill VDP and Instill Model. +
+ +
+ 💧 Instill VDP: AI pipeline builder for unstructured data
+ + **Instill VDP**, or **VDP (Versatile Data Pipeline)**, represents a comprehensive unstructured data infrastructure. Its purpose is to simplify the journey of processing unstructured data from start to finish: + + - **Extract:** Gather unstructured data from diverse sources, including AI applications, cloud/on-prem storage, and IoT devices. + - **Transform:** Utilize AI models to convert raw data into meaningful insights and actionable formats. + - **Load:** Efficiently move processed data to warehouses, applications, or other destinations. + + Embracing VDP is straightforward, whether you opt for Instill Cloud deployment or self-hosting via Instill Core. Consult our comprehensive [documentation](https://www.instill.tech/docs/core/vdp/deployment/overview) to delve into VDP deployment. +
+ +
+ ⚗️ Instill Model: Scalable AI model serving and training
+ + **Instill Model**, or simply **Model**, emerges as an advanced ModelOps platform. Here, the focus is on empowering you to seamlessly import, train and serve Machine Learning (ML) models for inference purposes. Like other projects, Instill Model's source code is available for your exploration. +
## No-/Low-code Access & Support diff --git a/charts/vdp/templates/_helpers.tpl b/charts/vdp/templates/_helpers.tpl index f3c270ec..db5998e0 100644 --- a/charts/vdp/templates/_helpers.tpl +++ b/charts/vdp/templates/_helpers.tpl @@ -77,31 +77,31 @@ app.kubernetes.io/name: {{ include "vdp.name" . }} {{- end -}} {{- end -}} -{{- define "base.database.host" -}} - {{- template "base.database" . -}} +{{- define "core.database.host" -}} + {{- template "core.database" . -}} {{- end -}} -{{- define "base.database.port" -}} +{{- define "core.database.port" -}} {{- print "5432" -}} {{- end -}} -{{- define "base.database.username" -}} +{{- define "core.database.username" -}} {{- print "postgres" -}} {{- end -}} -{{- define "base.database.rawPassword" -}} +{{- define "core.database.rawPassword" -}} {{- print "password" -}} {{- end -}} /*host:port*/ -{{- define "base.redis.addr" -}} +{{- define "core.redis.addr" -}} {{- with .Values.redis -}} - {{- default (printf "%s:6379" (include "base.redis" $ )) .addr -}} + {{- default (printf "%s:6379" (include "core.redis" $ )) .addr -}} {{- end -}} {{- end -}} -{{- define "base.mgmtBackend" -}} - {{- print "base-mgmt-backend" -}} +{{- define "core.mgmtBackend" -}} + {{- print "core-mgmt-backend" -}} {{- end -}} {{- define "vdp.pipelineBackend" -}} @@ -116,20 +116,20 @@ app.kubernetes.io/name: {{ include "vdp.name" . }} {{- printf "%s-controller-vdp" (include "vdp.fullname" .) -}} {{- end -}} -{{- define "base.database" -}} - {{- printf "base-database" -}} +{{- define "core.database" -}} + {{- printf "core-database" -}} {{- end -}} -{{- define "base.redis" -}} - {{- print "base-redis" -}} +{{- define "core.redis" -}} + {{- print "core-redis" -}} {{- end -}} -{{- define "base.temporal" -}} - {{- print "base-temporal" -}} +{{- define "core.temporal" -}} + {{- print "core-temporal" -}} {{- end -}} -{{- define "base.etcd" -}} - {{- print "base-etcd" -}} +{{- define "core.etcd" -}} + {{- print "core-etcd" -}} {{- end -}} {{/* pipeline service and container public port */}} @@ -158,46 +158,46 @@ app.kubernetes.io/name: {{ include "vdp.name" . }} {{- end -}} {{/* mgmt-backend service and container public port */}} -{{- define "base.mgmtBackend.publicPort" -}} +{{- define "core.mgmtBackend.publicPort" -}} {{- printf "8084" -}} {{- end -}} {{/* mgmt-backend service and container private port */}} -{{- define "base.mgmtBackend.privatePort" -}} +{{- define "core.mgmtBackend.privatePort" -}} {{- printf "3084" -}} {{- end -}} {{/* temporal container frontend gRPC port */}} -{{- define "base.temporal.frontend.grpcPort" -}} +{{- define "core.temporal.frontend.grpcPort" -}} {{- printf "7233" -}} {{- end -}} {{/* etcd port */}} -{{- define "base.etcd.clientPort" -}} +{{- define "core.etcd.clientPort" -}} {{- printf "2379" -}} {{- end -}} -{{- define "base.influxdb" -}} - {{- printf "base-influxdb2" -}} +{{- define "core.influxdb" -}} + {{- printf "core-influxdb2" -}} {{- end -}} -{{- define "base.influxdb.port" -}} +{{- define "core.influxdb.port" -}} {{- printf "8086" -}} {{- end -}} -{{- define "base.jaeger" -}} - {{- printf "base-jaeger-collector" -}} +{{- define "core.jaeger" -}} + {{- printf "core-jaeger-collector" -}} {{- end -}} -{{- define "base.jaeger.port" -}} +{{- define "core.jaeger.port" -}} {{- printf "14268" -}} {{- end -}} -{{- define "base.otel" -}} - {{- printf "base-opentelemetry-collector" -}} +{{- define "core.otel" -}} + {{- printf "core-opentelemetry-collector" -}} {{- end -}} -{{- define "base.otel.port" -}} +{{- define "core.otel.port" -}} {{- printf "8095" -}} {{- end -}} diff --git a/charts/vdp/templates/connector-backend/configmap.yaml b/charts/vdp/templates/connector-backend/configmap.yaml index 7829637a..a830ea2f 100644 --- a/charts/vdp/templates/connector-backend/configmap.yaml +++ b/charts/vdp/templates/connector-backend/configmap.yaml @@ -31,10 +31,10 @@ data: airbyte: /local excludelocalconnector: {{ .Values.connectorBackend.excludelocalconnector }} database: - username: {{ default (include "base.database.username" .) .Values.database.username }} - password: {{ default (include "base.database.rawPassword" .) .Values.database.password }} - host: {{ default (include "base.database.host" .) .Values.database.host }} - port: {{ default (include "base.database.port" .) .Values.database.port }} + username: {{ default (include "core.database.username" .) .Values.database.username }} + password: {{ default (include "core.database.rawPassword" .) .Values.database.password }} + host: {{ default (include "core.database.host" .) .Values.database.host }} + port: {{ default (include "core.database.port" .) .Values.database.port }} name: connector version: {{ .Values.connectorBackend.dbVersion }} timezone: Etc/UTC @@ -45,7 +45,7 @@ data: cache: redis: redisoptions: - addr: {{ default (include "base.redis.addr" .) .Values.redis.addr }} + addr: {{ default (include "core.redis.addr" .) .Values.redis.addr }} pipelinebackend: host: {{ template "vdp.pipelineBackend" . }} publicport: {{ template "vdp.pipelineBackend.publicPort" . }} @@ -55,8 +55,8 @@ data: key: /etc/instill-ai/vdp/ssl/pipeline/tls.key {{- end }} mgmtbackend: - host: {{ template "base.mgmtBackend" . }} - privateport: {{ template "base.mgmtBackend.privatePort" . }} + host: {{ template "core.mgmtBackend" . }} + privateport: {{ template "core.mgmtBackend.privatePort" . }} {{- if .Values.internalTLS.enabled }} https: cert: /etc/instill-ai/vdp/ssl/mgmt/tls.crt @@ -73,8 +73,8 @@ data: log: external: {{ .Values.tags.observability }} otelcollector: - host: {{ template "base.otel" . }} - port: {{ template "base.otel.port" . }} + host: {{ template "core.otel" . }} + port: {{ template "core.otel.port" . }} influxdb: url: {{ .Values.influxdbCloud.url }} token: {{ .Values.influxdbCloud.token }} diff --git a/charts/vdp/templates/connector-backend/deployment.yaml b/charts/vdp/templates/connector-backend/deployment.yaml index 7589873f..6f2df90a 100644 --- a/charts/vdp/templates/connector-backend/deployment.yaml +++ b/charts/vdp/templates/connector-backend/deployment.yaml @@ -52,9 +52,9 @@ spec: command: ['sh', '-c', "until pg_isready; do echo waiting for db; sleep 2; done"] env: - name: PGHOST - value: {{ template "base.database" . }} + value: {{ template "core.database" . }} - name: PGUSER - value: {{ template "base.database.username" . }} + value: {{ template "core.database.username" . }} {{- end }} - name: connector-backend-migration image: {{ .Values.connectorBackend.image.repository }}:{{ .Values.connectorBackend.image.tag }} @@ -116,9 +116,9 @@ spec: while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${MGMT_BACKEND_HOST}:${MGMT_BACKEND_PORT}/v1alpha/health/mgmt)" != "200" ]]; do echo waiting for mgmt-backend; sleep 1; done env: - name: MGMT_BACKEND_HOST - value: "{{ template "base.mgmtBackend" . }}" + value: "{{ template "core.mgmtBackend" . }}" - name: MGMT_BACKEND_PORT - value: "{{ template "base.mgmtBackend.publicPort" . }}" + value: "{{ template "core.mgmtBackend.publicPort" . }}" {{- with .Values.connectorBackend.extraInitContainers }} {{- toYaml . | indent 8 }} {{- end }} diff --git a/charts/vdp/templates/controller-vdp/configmap.yaml b/charts/vdp/templates/controller-vdp/configmap.yaml index 9f7757ab..ce99971f 100644 --- a/charts/vdp/templates/controller-vdp/configmap.yaml +++ b/charts/vdp/templates/controller-vdp/configmap.yaml @@ -19,8 +19,8 @@ data: key: /etc/instill-ai/vdp/ssl/controller/tls.key {{- end }} etcd: - host: {{ template "base.etcd" . }} - port: {{ template "base.etcd.clientPort" . }} + host: {{ template "core.etcd" . }} + port: {{ template "core.etcd.clientPort" . }} timeout: 5 connectorbackend: host: {{ template "vdp.connectorBackend" . }} @@ -41,9 +41,9 @@ data: key: /etc/instill-ai/vdp/ssl/pipeline/tls.key {{- end }} mgmtbackend: - host: {{ template "base.mgmtBackend" . }} - publicport: {{ template "base.mgmtBackend.publicPort" . }} - privateport: {{ template "base.mgmtBackend.privatePort" . }} + host: {{ template "core.mgmtBackend" . }} + publicport: {{ template "core.mgmtBackend.publicPort" . }} + privateport: {{ template "core.mgmtBackend.privatePort" . }} {{- if .Values.internalTLS.enabled }} https: cert: /etc/instill-ai/vdp/ssl/mgmt/tls.crt diff --git a/charts/vdp/templates/controller-vdp/deployment.yaml b/charts/vdp/templates/controller-vdp/deployment.yaml index 44a878e8..9cb74890 100644 --- a/charts/vdp/templates/controller-vdp/deployment.yaml +++ b/charts/vdp/templates/controller-vdp/deployment.yaml @@ -64,13 +64,13 @@ spec: - name: CONNECTOR_BACKEND_PORT value: "{{ template "vdp.connectorBackend.publicPort" . }}" - name: MGMT_BACKEND_HOST - value: "{{ template "base.mgmtBackend" . }}" + value: "{{ template "core.mgmtBackend" . }}" - name: MGMT_BACKEND_PORT - value: "{{ template "base.mgmtBackend.publicPort" . }}" + value: "{{ template "core.mgmtBackend.publicPort" . }}" - name: ETCD_HOST - value: "{{ template "base.etcd" . }}" + value: "{{ template "core.etcd" . }}" - name: ETCD_CLIENT_PORT - value: "{{ template "base.etcd.clientPort" . }}" + value: "{{ template "core.etcd.clientPort" . }}" containers: - name: controller image: {{ .Values.controllerVDP.image.repository }}:{{ .Values.controllerVDP.image.tag }} diff --git a/charts/vdp/templates/internal/auto-tls.yaml b/charts/vdp/templates/internal/auto-tls.yaml index e6a16281..ba1ebe76 100644 --- a/charts/vdp/templates/internal/auto-tls.yaml +++ b/charts/vdp/templates/internal/auto-tls.yaml @@ -6,7 +6,7 @@ {{- $connectorCrt := genSignedCert $connectorCN (list "127.0.0.1") (list "localhost" $connectorCN) 365 $ca }} {{- $modelCN := (include "vdp.model" .) }} {{- $modelCrt := genSignedCert $modelCN (list "127.0.0.1") (list "localhost" $modelCN) 365 $ca }} -{{- $mgmtCN := (include "base.mgmtBackend" .) }} +{{- $mgmtCN := (include "core.mgmtBackend" .) }} {{- $mgmtCrt := genSignedCert $mgmtCN (list "127.0.0.1") (list "localhost" $mgmtCN) 365 $ca }} {{- $consoleCN := (include "vdp.vdp.console" .) }} {{- $consoleCrt := genSignedCert $consoleCN (list "127.0.0.1") (list "localhost" $consoleCN) 365 $ca }} diff --git a/charts/vdp/templates/pipeline-backend/configmap.yaml b/charts/vdp/templates/pipeline-backend/configmap.yaml index 03ae95ef..dfa3ec53 100644 --- a/charts/vdp/templates/pipeline-backend/configmap.yaml +++ b/charts/vdp/templates/pipeline-backend/configmap.yaml @@ -34,8 +34,8 @@ data: key: /etc/instill-ai/vdp/ssl/connector/tls.key {{- end }} mgmtbackend: - host: {{ template "base.mgmtBackend" . }} - privateport: {{ template "base.mgmtBackend.privatePort" . }} + host: {{ template "core.mgmtBackend" . }} + privateport: {{ template "core.mgmtBackend.privatePort" . }} {{- if .Values.internalTLS.enabled }} https: cert: /etc/instill-ai/vdp/ssl/mgmt/tls.crt @@ -50,10 +50,10 @@ data: key: /etc/instill-ai/vdp/ssl/controller/tls.key {{- end }} database: - username: {{ default (include "base.database.username" .) .Values.database.username }} - password: {{ default (include "base.database.rawPassword" .) .Values.database.password }} - host: {{ default (include "base.database.host" .) .Values.database.host }} - port: {{ default (include "base.database.port" .) .Values.database.port }} + username: {{ default (include "core.database.username" .) .Values.database.username }} + password: {{ default (include "core.database.rawPassword" .) .Values.database.password }} + host: {{ default (include "core.database.host" .) .Values.database.host }} + port: {{ default (include "core.database.port" .) .Values.database.port }} name: pipeline version: {{ .Values.pipelineBackend.dbVersion }} timezone: Etc/UTC @@ -73,9 +73,9 @@ data: cache: redis: redisoptions: - addr: {{ default (include "base.redis.addr" .) .Values.redis.addr }} + addr: {{ default (include "core.redis.addr" .) .Values.redis.addr }} temporal: - hostport: {{ default (printf "%s-frontend-headless:%s" (include "base.temporal" .) (include "base.temporal.frontend.grpcPort" .)) .Values.pipelineBackend.temporal.hostPort }} + hostport: {{ default (printf "%s-frontend-headless:%s" (include "core.temporal" .) (include "core.temporal.frontend.grpcPort" .)) .Values.pipelineBackend.temporal.hostPort }} namespace: {{ default "pipeline-backend" .Values.pipelineBackend.temporal.namespace }} ca: {{ default "" .Values.pipelineBackend.temporal.ca }} cert: {{ default "" .Values.pipelineBackend.temporal.cert }} @@ -84,5 +84,5 @@ data: log: external: {{ .Values.tags.observability }} otelcollector: - host: {{ template "base.otel" . }} - port: {{ template "base.otel.port" . }} + host: {{ template "core.otel" . }} + port: {{ template "core.otel.port" . }} diff --git a/charts/vdp/templates/pipeline-backend/deployment.yaml b/charts/vdp/templates/pipeline-backend/deployment.yaml index cc71c5ee..c45e2299 100644 --- a/charts/vdp/templates/pipeline-backend/deployment.yaml +++ b/charts/vdp/templates/pipeline-backend/deployment.yaml @@ -55,9 +55,9 @@ spec: command: ['sh', '-c', "until pg_isready; do echo waiting for db; sleep 2; done"] env: - name: PGHOST - value: {{ template "base.database" . }} + value: {{ template "core.database" . }} - name: PGUSER - value: {{ template "base.database.username" . }} + value: {{ template "core.database.username" . }} {{- end }} {{- if not .Values.pipelineBackend.temporal.hostPort }} - name: temporal-admin-tools @@ -73,7 +73,7 @@ spec: if [[ ! $(tctl --namespace pipeline-backend namespace list | grep pipeline-backend) ]]; then tctl --namespace pipeline-backend namespace register; fi env: - name: TEMPORAL_CLI_ADDRESS - value: "{{ template "base.temporal" . }}-frontend:{{ template "base.temporal.frontend.grpcPort" . }}" + value: "{{ template "core.temporal" . }}-frontend:{{ template "core.temporal.frontend.grpcPort" . }}" {{- end }} - name: pipeline-backend-migration image: {{ .Values.pipelineBackend.image.repository }}:{{ .Values.pipelineBackend.image.tag }} @@ -100,9 +100,9 @@ spec: - name: CONNECTOR_BACKEND_PORT value: "{{ template "vdp.connectorBackend.publicPort" . }}" - name: MGMT_BACKEND_HOST - value: "{{ template "base.mgmtBackend" . }}" + value: "{{ template "core.mgmtBackend" . }}" - name: MGMT_BACKEND_PORT - value: "{{ template "base.mgmtBackend.publicPort" . }}" + value: "{{ template "core.mgmtBackend.publicPort" . }}" {{- with .Values.pipelineBackend.extraInitContainers }} {{- toYaml . | indent 8 }} {{- end }} diff --git a/charts/vdp/values.yaml b/charts/vdp/values.yaml index 2d7d1d7e..fbe11c4f 100644 --- a/charts/vdp/values.yaml +++ b/charts/vdp/values.yaml @@ -319,7 +319,7 @@ temporal: tolerations: [] affinity: {} influxdbCloud: - url: "http://base-influxdb2:8086" + url: "http://core-influxdb2:8086" token: "i-love-instill-ai" organization: "instill-ai" bucket: "instill-ai"