refactor(docker-compose,helm): retire connector-backend service (#395)

Because

- the connector-backend is highly coupled with the pipeline-backend, we
decided to merge them together to make the system structure more
elegant.

This commit

- retire connector-backend service

.env

@@ -5,7 +5,7 @@ COMPOSE_PROJECT_NAME=instill-vdp
 BUILD=false
 
 # docker compose profiles to selectively launch components for developing the latest codebase of the specified component.
-# the value can be all, api-gateway, mgmt, pipeline, connector, controller-vdp, model, controller-model, or console.
+# the value can be all, api-gateway, mgmt, pipeline, controller-vdp, model, controller-model, or console.
 PROFILE=all
 
 # system-wise config path (all core, vdp, and model projects must use the same path)
@@ -53,13 +53,6 @@ PIPELINE_BACKEND_HOST=pipeline-backend
 PIPELINE_BACKEND_PRIVATEPORT=3081
 PIPELINE_BACKEND_PUBLICPORT=8081
 
-# connector-backend
-CONNECTOR_BACKEND_IMAGE=instill/connector-backend
-CONNECTOR_BACKEND_VERSION=0.15.1-alpha
-CONNECTOR_BACKEND_HOST=connector-backend
-CONNECTOR_BACKEND_PRIVATEPORT=3082
-CONNECTOR_BACKEND_PUBLICPORT=8082
-
 # mgmt-backend
 MGMT_BACKEND_HOST=mgmt-backend
 MGMT_BACKEND_PRIVATEPORT=3084

.github/CONTRIBUTING.md

@@ -35,7 +35,7 @@ The env variable `PROFILE` is intended to specify which service component you wa
 
 - `{service}`
 
-  When you set `PROFILE={service}`, in which `{service}` can be `controller-vdp`, `connector` or `pipeline`, it means you want to develop on that particular service. The `make` command will launch the corresponding stack **WITHOUT** that service component and **WITH** all its dependencies. Given that, you can later on spin up and down the `{service}` in your dev container. Please take the [pipeline-backend](https://github.com/instill-ai/pipeline-backend#local-dev) as an example.
+  When you set `PROFILE={service}`, in which `{service}` can be `controller-vdp` or `pipeline`, it means you want to develop on that particular service. The `make` command will launch the corresponding stack **WITHOUT** that service component and **WITH** all its dependencies. Given that, you can later on spin up and down the `{service}` in your dev container. Please take the [pipeline-backend](https://github.com/instill-ai/pipeline-backend#local-dev) as an example.
 
 ### Tear down the local dev system
 

.github/workflows/helm-integration-test-backend.yml

@@ -97,11 +97,9 @@ jobs:
           helm install vdp charts/vdp --namespace instill-ai --create-namespace \
             --set edition=k8s-ce:test \
             --set pipelineBackend.image.tag=latest \
-            --set connectorBackend.image.tag=latest \
-            --set connectorBackend.excludelocalconnector=false \
+            --set pipelineBackend.excludelocalconnector=false \
             --set controllerVDP.image.tag=latest \
             --set tags.observability=false 
-          kubectl rollout status deployment vdp-connector-backend --namespace instill-ai
           kubectl rollout status deployment vdp-pipeline-backend --namespace instill-ai
           kubectl rollout status deployment vdp-controller-vdp --namespace instill-ai
           sleep 10
@@ -218,11 +216,9 @@ jobs:
           helm install vdp charts/vdp --namespace instill-ai --create-namespace \
             --set edition=k8s-ce:test \
             --set pipelineBackend.image.tag=latest \
-            --set connectorBackend.image.tag=latest \
-            --set connectorBackend.excludelocalconnector=false \
+            --set pipelineBackend.excludelocalconnector=false \
             --set controllerVDP.image.tag=latest \
             --set tags.observability=false
-          kubectl rollout status deployment vdp-connector-backend --namespace instill-ai --timeout=120s
           kubectl rollout status deployment vdp-pipeline-backend --namespace instill-ai --timeout=120s
           kubectl rollout status deployment vdp-controller-vdp --namespace instill-ai --timeout=120s
           sleep 10
@@ -331,11 +327,9 @@ jobs:
           helm install vdp charts/vdp --namespace instill-ai --create-namespace \
             --set edition=k8s-ce:test \
             --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
-            --set connectorBackend.image.tag=${CONNECTOR_BACKEND_VERSION} \
-            --set connectorBackend.excludelocalconnector=false \
+            --set pipelineBackend.excludelocalconnector=false \
             --set controllerVDP.image.tag=${CONTROLLER_VDP_VERSION} \
             --set tags.observability=false
-          kubectl rollout status deployment vdp-connector-backend --namespace instill-ai
           kubectl rollout status deployment vdp-pipeline-backend --namespace instill-ai
           kubectl rollout status deployment vdp-controller-vdp --namespace instill-ai
           sleep 10
@@ -458,11 +452,9 @@ jobs:
           helm install vdp charts/vdp --namespace instill-ai --create-namespace \
             --set edition=k8s-ce:test \
             --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
-            --set connectorBackend.image.tag=${CONNECTOR_BACKEND_VERSION} \
-            --set connectorBackend.excludelocalconnector=false \
+            --set pipelineBackend.excludelocalconnector=false \
             --set controllerVDP.image.tag=${CONTROLLER_VDP_VERSION} \
             --set tags.observability=false
-          kubectl rollout status deployment vdp-connector-backend --namespace instill-ai --timeout=120s
           kubectl rollout status deployment vdp-pipeline-backend --namespace instill-ai --timeout=120s
           kubectl rollout status deployment vdp-controller-vdp --namespace instill-ai --timeout=120s
           sleep 10

.github/workflows/helm-integration-test-latest.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        component: [pipeline-backend, connector-backend, controller-vdp]
+        component: [pipeline-backend, controller-vdp]
     uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@main
     with:
       component: ${{ matrix.component }}

.github/workflows/helm-integration-test-release.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        component: [pipeline-backend, connector-backend, controller-vdp]
+        component: [pipeline-backend, controller-vdp]
     uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@main
     with:
       component: ${{ matrix.component }}

.github/workflows/integration-test-latest.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        component: [pipeline-backend, connector-backend, controller-vdp]
+        component: [pipeline-backend, controller-vdp]
     uses: instill-ai/vdp/.github/workflows/integration-test-backend.yml@main
     with:
       component: ${{ matrix.component }}

.github/workflows/integration-test-release.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        component: [pipeline-backend, connector-backend, controller-vdp]
+        component: [pipeline-backend, controller-vdp]
     uses: instill-ai/vdp/.github/workflows/integration-test-backend.yml@main
     with:
       component: ${{ matrix.component }}

.github/workflows/make-all.yml

@@ -58,7 +58,6 @@ jobs:
         run: |
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/core/v1alpha/health/mgmt
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector
 
       - name: Tear down Instill VDP (release)
         run: |

.github/workflows/make-latest.yml

@@ -62,7 +62,6 @@ jobs:
         run: |
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/core/v1alpha/health/mgmt
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3085/v1alpha/health/controller
 
       - name: Tear down Instill VDP (latest)

.github/workflows/releases.yml

@@ -76,7 +76,6 @@ jobs:
         run: |
           source .env
           yq ".pipelineBackend.image.tag=\"$PIPELINE_BACKEND_VERSION\"" -i charts/vdp/values.yaml
-          yq ".connectorBackend.image.tag=\"$CONNECTOR_BACKEND_VERSION\"" -i charts/vdp/values.yaml
           yq ".controllerVDP.image.tag=\"$CONTROLLER_VDP_VERSION\"" -i charts/vdp/values.yaml
       - name: Get current chart version
         id: get-current-chart-version

Dockerfile

@@ -33,7 +33,6 @@ RUN git clone https://github.com/instill-ai/core.git
 WORKDIR /instill-ai/vdp
 
 RUN git clone https://github.com/instill-ai/pipeline-backend.git
-RUN git clone https://github.com/instill-ai/connector-backend.git
 RUN git clone https://github.com/instill-ai/controller-vdp.git
 
 FROM alpine:${ALPINE_VERSION} AS release
@@ -53,7 +52,6 @@ RUN git clone -b v${INSTILL_CORE_VERSION} -c advice.detachedHead=false https://g
 
 WORKDIR /instill-ai/vdp
 
-ARG PIPELINE_BACKEND_VERSION CONNECTOR_BACKEND_VERSION CONTROLLER_VDP_VERSION
+ARG PIPELINE_BACKEND_VERSION CONTROLLER_VDP_VERSION
 RUN git clone -b v${PIPELINE_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/pipeline-backend.git
-RUN git clone -b v${CONNECTOR_BACKEND_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/connector-backend.git
 RUN git clone -b v${CONTROLLER_VDP_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/controller-vdp.git

Makefile

@@ -31,7 +31,6 @@ all:			## Launch all services with their up-to-date release version
 			--build-arg CACHE_DATE="$(shell date)" \
 			--build-arg INSTILL_CORE_VERSION=${INSTILL_CORE_VERSION} \
 			--build-arg PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \
-			--build-arg CONNECTOR_BACKEND_VERSION=${CONNECTOR_BACKEND_VERSION} \
 			--build-arg CONTROLLER_VDP_VERSION=${CONTROLLER_VDP_VERSION} \
 			--target release \
 			-t ${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} .; \
@@ -165,7 +164,6 @@ build-latest:				## Build latest images for all VDP components
 		--name ${CONTAINER_BUILD_NAME}-latest \
 		${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
 			PIPELINE_BACKEND_VERSION=latest \
-			CONNECTOR_BACKEND_VERSION=latest \
 			CONTROLLER_VDP_VERSION=latest \
 			docker compose -f docker-compose.build.yml build --progress plain \
 		"
@@ -179,7 +177,6 @@ build-release:				## Build release images for all VDP components
 		--build-arg CACHE_DATE="$(shell date)" \
 		--build-arg INSTILL_CORE_VERSION=${INSTILL_CORE_VERSION} \
 		--build-arg PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \
-		--build-arg CONNECTOR_BACKEND_VERSION=${CONNECTOR_BACKEND_VERSION} \
 		--build-arg CONTROLLER_VDP_VERSION=${CONTROLLER_VDP_VERSION} \
 		--target release \
 		-t ${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} .
@@ -191,7 +188,6 @@ build-release:				## Build release images for all VDP components
 		${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} /bin/sh -c " \
 			INSTILL_CORE_VERSION=${INSTILL_CORE_VERSION} \
 			PIPELINE_BACKEND_VERSION=${PIPELINE_BACKEND_VERSION} \
-			CONNECTOR_BACKEND_VERSION=${CONNECTOR_BACKEND_VERSION} \
 			CONTROLLER_VDP_VERSION=${CONTROLLER_VDP_VERSION} \
 			docker compose -f docker-compose.build.yml build --progress plain \
 		"
@@ -204,7 +200,6 @@ integration-test-latest:			## Run integration test on the latest VDP
 		--name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-latest \
 		${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' \
 		"
 	@make down
@@ -217,14 +212,13 @@ integration-test-release:			## Run integration test on the release VDP
 		--name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-release \
 		${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=${API_GATEWAY_HOST}:${API_GATEWAY_PORT}' \
 		"
 	@make down
 
 .PHONY: helm-integration-test-latest
 helm-integration-test-latest:                       ## Run integration test on the Helm latest for VDP
-	# @make build-latest
+	@make build-latest
 	@export TMP_CONFIG_DIR=$(shell mktemp -d) && docker run --rm \
 		-v ${HOME}/.kube/config:/root/.kube/config \
 		-v /var/run/docker.sock:/var/run/docker.sock \
@@ -246,31 +240,27 @@ helm-integration-test-latest:                       ## Run integration test on t
 					--set tags.prometheusStack=false' \
 			/bin/sh -c 'rm -rf $${TMP_CONFIG_DIR}/*' \
 		" && rm -rf $${TMP_CONFIG_DIR}
-	@kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s
+	@kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=180s
 	@export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o jsonpath="{.items[0].metadata.name}") && \
 		kubectl --namespace ${HELM_NAMESPACE} port-forward $${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 &
 	@while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done
 	@helm install ${HELM_RELEASE_NAME} charts/vdp --namespace ${HELM_NAMESPACE} --create-namespace \
 		--set edition=k8s-ce:test \
 		--set pipelineBackend.image.tag=latest \
-		--set connectorBackend.image.tag=latest \
-		--set connectorBackend.excludelocalconnector=false \
+		--set pipelineBackend.excludelocalconnector=false \
 		--set controllerVDP.image.tag=latest \
 		--set tags.observability=false
-	@kubectl rollout status deployment vdp-connector-backend --namespace ${HELM_NAMESPACE} --timeout=120s
 	@kubectl rollout status deployment vdp-pipeline-backend --namespace ${HELM_NAMESPACE} --timeout=120s
 	@kubectl rollout status deployment vdp-controller-vdp --namespace ${HELM_NAMESPACE} --timeout=120s
 	@sleep 30
 ifeq ($(UNAME_S),Darwin)
 	@docker run --rm --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-helm-latest ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' \
 		"
 else ifeq ($(UNAME_S),Linux)
 	@docker run --rm --network host --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-helm-latest ${CONTAINER_COMPOSE_IMAGE_NAME}:latest /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' \
 		"
 endif
@@ -304,31 +294,27 @@ helm-integration-test-release:                       ## Run integration test on
 					--set tags.observability=false \
 					--set tags.prometheusStack=false' \
 		"
-	@kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=120s
+	@kubectl rollout status deployment core-api-gateway --namespace ${HELM_NAMESPACE} --timeout=180s
 	@export API_GATEWAY_POD_NAME=$$(kubectl get pods --namespace ${HELM_NAMESPACE} -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=core" -o jsonpath="{.items[0].metadata.name}") && \
 		kubectl --namespace ${HELM_NAMESPACE} port-forward $${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 &
 	@while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done
 	@helm install ${HELM_RELEASE_NAME} charts/vdp --namespace ${HELM_NAMESPACE} --create-namespace \
 		--set edition=k8s-ce:test \
 		--set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
-		--set connectorBackend.image.tag=${CONNECTOR_BACKEND_VERSION} \
-		--set connectorBackend.excludelocalconnector=false \
+		--set pipelineBackend.excludelocalconnector=false \
 		--set controllerVDP.image.tag=${CONTROLLER_VDP_VERSION} \
 		--set tags.observability=false
-	@kubectl rollout status deployment vdp-connector-backend --namespace ${HELM_NAMESPACE} --timeout=120s
 	@kubectl rollout status deployment vdp-pipeline-backend --namespace ${HELM_NAMESPACE} --timeout=120s
 	@kubectl rollout status deployment vdp-controller-vdp --namespace ${HELM_NAMESPACE} --timeout=120s
 	@sleep 10
 ifeq ($(UNAME_S),Darwin)
 	@docker run --rm --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-helm-release ${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' \
 		"
 else ifeq ($(UNAME_S),Linux)
 	@docker run --rm --network host --name ${CONTAINER_BACKEND_INTEGRATION_TEST_NAME}-helm-release ${CONTAINER_COMPOSE_IMAGE_NAME}:${INSTILL_VDP_VERSION} /bin/sh -c " \
 			/bin/sh -c 'cd pipeline-backend && make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}' && \
-			/bin/sh -c 'cd connector-backend && make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}' && \
 			/bin/sh -c 'cd controller-vdp && make integration-test API_GATEWAY_URL=host.docker.internal:${API_GATEWAY_PORT}' \
 		"
 endif

charts/vdp/templates/_helpers.tpl

@@ -108,9 +108,6 @@ app.kubernetes.io/name: {{ include "vdp.name" . }}
   {{- printf "%s-pipeline-backend" (include "vdp.fullname" .) -}}
 {{- end -}}
 
-{{- define "vdp.connectorBackend" -}}
-  {{- printf "%s-connector-backend" (include "vdp.fullname" .) -}}
-{{- end -}}
 
 {{- define "vdp.controllerVDP" -}}
   {{- printf "%s-controller-vdp" (include "vdp.fullname" .) -}}
@@ -142,16 +139,6 @@ app.kubernetes.io/name: {{ include "vdp.name" . }}
   {{- printf "3081" -}}
 {{- end -}}
 
-{{/* connector service and container public port */}}
-{{- define "vdp.connectorBackend.publicPort" -}}
-  {{- printf "8082" -}}
-{{- end -}}
-
-{{/* connector service and container private port */}}
-{{- define "vdp.connectorBackend.privatePort" -}}
-  {{- printf "3082" -}}
-{{- end -}}
-
 {{/* controller service and container private port */}}
 {{- define "vdp.controllerVDP.privatePort" -}}
   {{- printf "3085" -}}
@@ -209,14 +196,6 @@ app.kubernetes.io/name: {{ include "vdp.name" . }}
   {{- end -}}
 {{- end -}}
 
-{{- define "vdp.internalTLS.connectorBackend.secretName" -}}
-  {{- if eq .Values.internalTLS.certSource "secret" -}}
-    {{- .Values.internalTLS.connectorBackend.secretName -}}
-  {{- else -}}
-    {{- printf "%s-connector-backend-internal-tls" (include "vdp.fullname" .) -}}
-  {{- end -}}
-{{- end -}}
-
 {{- define "vdp.internalTLS.controllerVDP.secretName" -}}
   {{- if eq .Values.internalTLS.certSource "secret" -}}
     {{- .Values.internalTLS.controllerVDP.secretName -}}