From 9f233510a05147c8208462bca76fd85a594b5032 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 10 Dec 2024 22:55:57 +0900 Subject: [PATCH 1/7] :bug: fix minikube start failure --- Makefile.d/e2e.mk | 4 ++-- Makefile.d/minikube.mk | 7 +++++-- Makefile.d/tools.mk | 8 ++++++++ dockers/ci/base/Dockerfile | 2 ++ dockers/dev/Dockerfile | 1 + hack/docker/gen/main.go | 3 ++- 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index d9502708c1..ad373d6740 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -154,8 +154,8 @@ e2e/actions/run/readreplica: \ $(MAKE) k8s/vald/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 20 - kubectl wait --for=condition=Ready pod -l "app=$(AGENT_NGT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) - kubectl wait --for=condition=ContainersReady pod -l "app=$(AGENT_NGT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) + kubectl wait --for=condition=Ready pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) + kubectl wait --for=condition=ContainersReady pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) $(MAKE) k8s/vald-readreplica/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 3 diff --git a/Makefile.d/minikube.mk b/Makefile.d/minikube.mk index e668e02630..5d86ab061c 100644 --- a/Makefile.d/minikube.mk +++ b/Makefile.d/minikube.mk @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # + .PHONY: minikube/install minikube/install: $(BINDIR)/minikube @@ -24,12 +25,14 @@ $(BINDIR)/minikube: # Start minikube with CSI Driver and Volume Snapshots support # Only use this for development related to Volume Snapshots. Usually k3d is faster. .PHONY: minikube/start -minikube/start: - minikube start --force +minikube/start: \ + $(BINDIR)/docker + minikube start --driver=docker --force minikube addons enable volumesnapshots minikube addons enable csi-hostpath-driver minikube addons disable storage-provisioner minikube addons disable default-storageclass + minikube addons enable metrics-server kubectl patch storageclass csi-hostpath-sc -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' .PHONY: minikube/delete diff --git a/Makefile.d/tools.mk b/Makefile.d/tools.mk index 0c7e7d98c3..88ba4a26b4 100644 --- a/Makefile.d/tools.mk +++ b/Makefile.d/tools.mk @@ -272,3 +272,11 @@ $(BINDIR)/yq: && curl -fsSL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(subst x86_64,amd64,$(shell echo $(DARCH) | tr '[:upper:]' '[:lower:]')) -o $(BINDIR)/yq \ && chmod a+x $(BINDIR)/yq +.PHONY: docker-cli/install +docker-cli/install: $(BINDIR)/docker + +$(BINDIR)/docker: $(BINDIR) + curl -fsSL https://download.docker.com/linux/static/stable/$(shell uname -m)/docker-$(shell echo $(DOCKER_VERSION) | cut -c2-).tgz -o $(TEMP_DIR)/docker.tgz \ + && tar -xzvf $(TEMP_DIR)/docker.tgz -C $(TEMP_DIR) \ + && mv $(TEMP_DIR)/docker/docker $(BINDIR) \ + && rm -rf $(TEMP_DIR)/docker{.tgz,} diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index 85276f07b2..49ae0517a2 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -72,6 +72,7 @@ RUN --mount=type=bind,target=.,rw \ locales \ git \ npm \ + sudo \ cmake \ g++ \ gcc \ @@ -122,6 +123,7 @@ RUN --mount=type=bind,target=.,rw \ && make telepresence/install \ && make tparse/install \ && make yq/install \ + && make docker-cli/install \ && make ngt/install \ && make faiss/install \ && make usearch/install \ diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 8c5b08dcd3..8a0d0f7f69 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -137,6 +137,7 @@ RUN --mount=type=bind,target=.,rw \ && make telepresence/install \ && make tparse/install \ && make yq/install \ + && make docker-cli/install \ && make ngt/install \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 4ba68db4a0..77242e7b24 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -508,6 +508,7 @@ var ( "make telepresence/install", "make tparse/install", "make yq/install", + "make docker-cli/install", } devContainerPreprocess = []string{ @@ -807,7 +808,7 @@ func main() { ContainerType: CIContainer, PackageDir: "ci/base", RuntimeUser: defaultBuildUser, - ExtraPackages: append([]string{"npm"}, append(clangBuildDeps, + ExtraPackages: append([]string{"npm", "sudo"}, append(clangBuildDeps, append(ngtBuildDeps, append(faissBuildDeps, append(rustBuildDeps, From e34f61ef63bf0ae81874ce486fdb96e139f7342c Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Wed, 11 Dec 2024 10:13:56 +0900 Subject: [PATCH 2/7] add docker option --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 774161c298..e9d9c9b0a4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -304,7 +304,7 @@ jobs: timeout-minutes: 60 container: image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - options: "--add-host host.docker.internal:host-gateway" + options: "--net host --add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config From 18996f6c9fe6c17d27caf77751b0404d57edc04f Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Fri, 13 Dec 2024 11:10:45 +0900 Subject: [PATCH 3/7] use kind --- .github/workflows/e2e.yaml | 2 +- Makefile.d/e2e.mk | 15 +++++++++++--- Makefile.d/kind.mk | 40 +++++++++++++++++++++++++++++++++++--- Makefile.d/minikube.mk | 2 +- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e9d9c9b0a4..774161c298 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -304,7 +304,7 @@ jobs: timeout-minutes: 60 container: image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }} - options: "--net host --add-host host.docker.internal:host-gateway" + options: "--add-host host.docker.internal:host-gateway" steps: - uses: actions/checkout@v4 - name: Set Git config diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index ad373d6740..fd5f05a7d7 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -145,12 +145,21 @@ e2e/actions/run/job: \ ## run GitHub Actions E2E test (Stream CRUD with read replica ) e2e/actions/run/readreplica: \ hack/benchmark/assets/dataset/$(E2E_DATASET_NAME) \ - minikube/restart + kind/vs/restart \ + k8s/metrics/metrics-server/deploy + helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server -n kube-system kubectl wait -n kube-system --for=condition=Available deployment/metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) sleep 2 - kubectl wait -n kube-system --for=condition=Ready pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) - kubectl wait -n kube-system --for=condition=ContainersReady pod -l k8s-app=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) + kubectl wait -n kube-system --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) + kubectl wait -n kube-system --for=condition=ContainersReady pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) sleep 3 + mkdir -p $(TEMP_DIR)/csi-driver-hostpath \ + && curl -fsSL https://github.com/kubernetes-csi/csi-driver-host-path/archive/refs/tags/v1.15.0.tar.gz | tar zxf - -C $(TEMP_DIR)/csi-driver-hostpath --strip-components 1 \ + && cd $(TEMP_DIR)/csi-driver-hostpath \ + && deploy/kubernetes-latest/deploy.sh \ + && kubectl apply -f ./examples/csi-storageclass.yaml \ + && kubectl apply -f ././examples/csi-pvc.yaml \ + && rm -rf $(TEMP_DIR)/csi-driver-hostpath $(MAKE) k8s/vald/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 20 diff --git a/Makefile.d/kind.mk b/Makefile.d/kind.mk index b7ca9605c1..8eca31511a 100644 --- a/Makefile.d/kind.mk +++ b/Makefile.d/kind.mk @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +SNAPSHOTTER_VERSION=v8.2.0 + .PHONY: kind/install ## install KinD kind/install: $(BINDIR)/kind @@ -25,7 +28,8 @@ $(BINDIR)/kind: .PHONY: kind/start ## start kind (kubernetes in docker) cluster -kind/start: +kind/start: \ + $(BINDIR)/docker kind create cluster --name $(NAME) @make kind/login @@ -36,7 +40,8 @@ kind/login: .PHONY: kind/stop ## stop kind (kubernetes in docker) cluster -kind/stop: +kind/stop: \ + $(BINDIR)/docker kind delete cluster --name $(NAME) .PHONY: kind/restart @@ -54,7 +59,6 @@ kind/cluster/start: kubectl apply -f https://projectcontour.io/quickstart/operator.yaml kubectl apply -f https://projectcontour.io/quickstart/contour-custom-resource.yaml - .PHONY: kind/cluster/stop ## stop kind (kubernetes in docker) multi node cluster kind/cluster/stop: @@ -70,3 +74,33 @@ kind/cluster/login: kind/cluster/restart: \ kind/cluster/stop \ kind/cluster/start + +.PHONY: kind/vs/start +## start kind (kubernetes in docker) cluster with volume snapshot +kind/vs/start: \ + $(BINDIR)/docker + sed -e 's/apiServerAddress: "127.0.0.1"/apiServerAddress: "$(shell grep host.docker.internal /etc/hosts | cut -f1)"/' $(ROOTDIR)/k8s/debug/kind/e2e.yaml | kind create cluster --name $(NAME)-vs --config - + + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + + @make kind/vs/login + +.PHONY: kind/vs/stop +## stop kind (kubernetes in docker) cluster with volume snapshot +kind/vs/stop: + kind delete cluster --name $(NAME)-vs + +.PHONY: kind/vs/login +## login command for kind (kubernetes in docker) cluster with volume snapshot +kind/vs/login: + kubectl cluster-info --context kind-$(NAME)-vs + +.PHONY: kind/vs/restart +## restart kind (kubernetes in docker) cluster with volume snapshot +kind/vs/restart: \ + kind/vs/stop \ + kind/vs/start \ No newline at end of file diff --git a/Makefile.d/minikube.mk b/Makefile.d/minikube.mk index 5d86ab061c..ebec7bc81f 100644 --- a/Makefile.d/minikube.mk +++ b/Makefile.d/minikube.mk @@ -27,7 +27,7 @@ $(BINDIR)/minikube: .PHONY: minikube/start minikube/start: \ $(BINDIR)/docker - minikube start --driver=docker --force + minikube start --driver=docker --force --alsologtostderr -v=1 minikube addons enable volumesnapshots minikube addons enable csi-hostpath-driver minikube addons disable storage-provisioner From dd9b8058ce60d6cf98567ffd64bf00ce0c846d2d Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Fri, 13 Dec 2024 14:47:21 +0900 Subject: [PATCH 4/7] fix workflow --- .github/actions/setup-e2e/action.yaml | 10 ++++++++++ .github/workflows/e2e.yaml | 2 +- Makefile.d/e2e.mk | 11 +---------- Makefile.d/kind.mk | 19 +++++++++++++++---- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index 52fa1f7330..66263b1cbe 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -36,6 +36,10 @@ inputs: description: "If minikube is not required, set this to true and set require_k3d to false" required: false default: "false" + require_kind: + description: "If kind is not required, set this to true and set require_k3d to false" + required: false + default: "false" ingress_port: description: 'If it is not "0", ingress will be exposed to the specified port' required: false @@ -91,6 +95,12 @@ runs: run: | make minikube/install make minikube/start + - name: Setup kind environment + if: ${{ inputs.require_kind == 'true' }} + shell: bash + run: | + make kind/install + make kind/vs/start - name: Check Kubernetes cluster shell: bash run: | diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 774161c298..a5b41c79a4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -315,7 +315,7 @@ jobs: uses: ./.github/actions/setup-e2e with: require_k3d: "false" - require_minikube: "true" + require_kind: "true" - name: Deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index fd5f05a7d7..ad38fcda82 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -145,21 +145,12 @@ e2e/actions/run/job: \ ## run GitHub Actions E2E test (Stream CRUD with read replica ) e2e/actions/run/readreplica: \ hack/benchmark/assets/dataset/$(E2E_DATASET_NAME) \ - kind/vs/restart \ - k8s/metrics/metrics-server/deploy - helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server -n kube-system + kind/vs/restart kubectl wait -n kube-system --for=condition=Available deployment/metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) sleep 2 kubectl wait -n kube-system --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait -n kube-system --for=condition=ContainersReady pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) sleep 3 - mkdir -p $(TEMP_DIR)/csi-driver-hostpath \ - && curl -fsSL https://github.com/kubernetes-csi/csi-driver-host-path/archive/refs/tags/v1.15.0.tar.gz | tar zxf - -C $(TEMP_DIR)/csi-driver-hostpath --strip-components 1 \ - && cd $(TEMP_DIR)/csi-driver-hostpath \ - && deploy/kubernetes-latest/deploy.sh \ - && kubectl apply -f ./examples/csi-storageclass.yaml \ - && kubectl apply -f ././examples/csi-pvc.yaml \ - && rm -rf $(TEMP_DIR)/csi-driver-hostpath $(MAKE) k8s/vald/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 20 diff --git a/Makefile.d/kind.mk b/Makefile.d/kind.mk index 8eca31511a..271acaf2cd 100644 --- a/Makefile.d/kind.mk +++ b/Makefile.d/kind.mk @@ -50,7 +50,6 @@ kind/restart: \ kind/stop \ kind/start - .PHONY: kind/cluster/start ## start kind (kubernetes in docker) multi node cluster kind/cluster/start: @@ -80,6 +79,7 @@ kind/cluster/restart: \ kind/vs/start: \ $(BINDIR)/docker sed -e 's/apiServerAddress: "127.0.0.1"/apiServerAddress: "$(shell grep host.docker.internal /etc/hosts | cut -f1)"/' $(ROOTDIR)/k8s/debug/kind/e2e.yaml | kind create cluster --name $(NAME)-vs --config - + @make kind/vs/login kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -87,11 +87,22 @@ kind/vs/start: \ kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(SNAPSHOTTER_VERSION)/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml - @make kind/vs/login + mkdir -p $(TEMP_DIR)/csi-driver-hostpath \ + && curl -fsSL https://github.com/kubernetes-csi/csi-driver-host-path/archive/refs/tags/v1.15.0.tar.gz | tar zxf - -C $(TEMP_DIR)/csi-driver-hostpath --strip-components 1 \ + && cd $(TEMP_DIR)/csi-driver-hostpath \ + && deploy/kubernetes-latest/deploy.sh \ + && kubectl apply -f ./examples/csi-storageclass.yaml \ + && kubectl apply -f ././examples/csi-pvc.yaml \ + && rm -rf $(TEMP_DIR)/csi-driver-hostpath + + @make k8s/metrics/metrics-server/deploy + helm upgrade --install --set args={--kubelet-insecure-tls} metrics-server metrics-server/metrics-server -n kube-system + sleep $(K8S_SLEEP_DURATION_FOR_WAIT_COMMAND) .PHONY: kind/vs/stop ## stop kind (kubernetes in docker) cluster with volume snapshot -kind/vs/stop: +kind/vs/stop: \ + $(BINDIR)/docker kind delete cluster --name $(NAME)-vs .PHONY: kind/vs/login @@ -103,4 +114,4 @@ kind/vs/login: ## restart kind (kubernetes in docker) cluster with volume snapshot kind/vs/restart: \ kind/vs/stop \ - kind/vs/start \ No newline at end of file + kind/vs/start From bde000e3dc779fb4a85ff65157545418d654f155 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Fri, 13 Dec 2024 16:20:07 +0900 Subject: [PATCH 5/7] fix unintended ignore directory --- .../e2e-deploy-vald-readreplica/action.yaml | 4 ---- .github/actions/e2e-deploy-vald/action.yaml | 7 +++++-- .github/helm/values/values-readreplica.yaml | 2 ++ .github/workflows/e2e.yaml | 15 +++++++------- .gitignore | 4 ++-- Makefile.d/e2e.mk | 8 ++++---- Makefile.d/k8s.mk | 6 ++++-- k8s/debug/kind/e2e.yaml | 20 +++++++++++++++++++ 8 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 k8s/debug/kind/e2e.yaml diff --git a/.github/actions/e2e-deploy-vald-readreplica/action.yaml b/.github/actions/e2e-deploy-vald-readreplica/action.yaml index cfcdad536e..8469a54ed8 100644 --- a/.github/actions/e2e-deploy-vald-readreplica/action.yaml +++ b/.github/actions/e2e-deploy-vald-readreplica/action.yaml @@ -69,8 +69,6 @@ runs: sleep 3 - kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} - kubectl get pods podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'` @@ -89,8 +87,6 @@ runs: sleep 3 - kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} - kubectl get pods podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'` diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 63ffae0742..811f163ea5 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -79,7 +79,8 @@ runs: sleep 3 - kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} + kubectl wait --for=condition=Ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} + kubectl wait --for=condition=ContainersReady pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} kubectl get pods @@ -99,7 +100,9 @@ runs: sleep 3 - kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} + kubectl get pods + + kubectl wait --for=condition=Ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} kubectl get pods diff --git a/.github/helm/values/values-readreplica.yaml b/.github/helm/values/values-readreplica.yaml index 6cf28a3de1..7e2d32fe36 100644 --- a/.github/helm/values/values-readreplica.yaml +++ b/.github/helm/values/values-readreplica.yaml @@ -59,6 +59,8 @@ agent: snapshot_classname: "csi-hostpath-snapclass" hpa: enabled: true + name: vald-agent-readreplica + volume_name: vald-agent-readreplica-pvc discoverer: minReplicas: 1 hpa: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a5b41c79a4..519927f499 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -316,16 +316,15 @@ jobs: with: require_k3d: "false" require_kind: "true" - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-readreplica.yaml - wait_for_selector: app=vald-agent - name: Deploy Vald Read Replica id: deploy_vald_readreplica uses: ./.github/actions/e2e-deploy-vald-readreplica + with: + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-readreplica.yaml + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-readreplica.yaml @@ -347,7 +346,7 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e/readreplica env: - POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} e2e-stream-crud-with-mirror: name: "E2E test (Stream CRUD) with mirror" needs: [detect-ci-container] diff --git a/.gitignore b/.gitignore index ea7be3983c..41c50e8741 100644 --- a/.gitignore +++ b/.gitignore @@ -59,8 +59,8 @@ telepresence.log # Generated by Cargo # will have compiled files and executables -debug/ -target/ +rust/debug/ +rust/target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index ad38fcda82..92c74f3dcf 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -151,14 +151,14 @@ e2e/actions/run/readreplica: \ kubectl wait -n kube-system --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait -n kube-system --for=condition=ContainersReady pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) sleep 3 + $(MAKE) k8s/vald-readreplica/deploy \ + HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml + sleep 3 $(MAKE) k8s/vald/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 20 kubectl wait --for=condition=Ready pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait --for=condition=ContainersReady pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) - $(MAKE) k8s/vald-readreplica/deploy \ - HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml - sleep 3 kubectl wait --for=condition=Ready pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait --for=condition=ContainersReady pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl get pods @@ -166,7 +166,7 @@ e2e/actions/run/readreplica: \ echo $$pod_name; \ $(MAKE) E2E_TARGET_POD_NAME=$$pod_name e2e/readreplica $(MAKE) k8s/vald/delete - $(MAKE) minikube/delete + $(MAKE) kind/vs/stop .PHONY: e2e/actions/run/stream/crud/skip ## run GitHub Actions E2E test (Stream CRUD with SkipExistsCheck = true) diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index e2dbf87495..dfd1522718 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -262,8 +262,10 @@ k8s/vald-readreplica/deploy: --output-dir $(TEMP_DIR) \ charts/vald-readreplica kubectl apply -f $(TEMP_DIR)/vald-readreplica/templates - sleep 2 - kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=vald-readreplica --timeout=600s + sleep 5 + + # kubectl get pods + # kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=vald-readreplica --timeout=600s .PHONY: k8s/vald-readreplica/delete ## delete vald-helm-operator from k8s diff --git a/k8s/debug/kind/e2e.yaml b/k8s/debug/kind/e2e.yaml new file mode 100644 index 0000000000..292ddde74b --- /dev/null +++ b/k8s/debug/kind/e2e.yaml @@ -0,0 +1,20 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + apiServerAddress: "127.0.0.1" + apiServerPort: 6443 From 9ab90c2622e528e202d0c2197f021c7071340086 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 17 Dec 2024 21:32:45 +0900 Subject: [PATCH 6/7] Update Makefile.d/kind.mk Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Kosuke Morimoto --- .../e2e-deploy-vald-readreplica/action.yaml | 4 ++ .github/actions/e2e-deploy-vald/action.yaml | 3 -- .github/helm/values/values-readreplica.yaml | 2 +- .github/workflows/e2e.yaml | 6 --- Makefile.d/e2e.mk | 6 --- Makefile.d/k8s.mk | 49 ++++++++++--------- Makefile.d/kind.mk | 4 +- Makefile.d/minikube.mk | 5 +- charts/vald/templates/_helpers.tpl | 4 ++ 9 files changed, 38 insertions(+), 45 deletions(-) diff --git a/.github/actions/e2e-deploy-vald-readreplica/action.yaml b/.github/actions/e2e-deploy-vald-readreplica/action.yaml index 8469a54ed8..eb8f4b1c79 100644 --- a/.github/actions/e2e-deploy-vald-readreplica/action.yaml +++ b/.github/actions/e2e-deploy-vald-readreplica/action.yaml @@ -85,10 +85,14 @@ runs: run: | make k8s/vald-readreplica/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + stern vald-lb-gateway & + stern vald-agent-readreplica & sleep 3 kubectl get pods + kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=600s + podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'` echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT env: diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 811f163ea5..d518146472 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -97,11 +97,8 @@ runs: if: ${{ inputs.use_local_charts == 'true' }} run: | make k8s/vald/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" - sleep 3 - kubectl get pods - kubectl wait --for=condition=Ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} kubectl get pods diff --git a/.github/helm/values/values-readreplica.yaml b/.github/helm/values/values-readreplica.yaml index 7e2d32fe36..7f30e8ebc9 100644 --- a/.github/helm/values/values-readreplica.yaml +++ b/.github/helm/values/values-readreplica.yaml @@ -16,7 +16,7 @@ defaults: logging: - level: info + level: debug networkPolicy: enabled: true gateway: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 519927f499..9edada1419 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -319,12 +319,6 @@ jobs: - name: Deploy Vald Read Replica id: deploy_vald_readreplica uses: ./.github/actions/e2e-deploy-vald-readreplica - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-readreplica.yaml - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald with: helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-readreplica.yaml diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk index 92c74f3dcf..fd85b092dd 100644 --- a/Makefile.d/e2e.mk +++ b/Makefile.d/e2e.mk @@ -150,15 +150,9 @@ e2e/actions/run/readreplica: \ sleep 2 kubectl wait -n kube-system --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait -n kube-system --for=condition=ContainersReady pod -l app.kubernetes.io/name=metrics-server --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) - sleep 3 $(MAKE) k8s/vald-readreplica/deploy \ HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml sleep 3 - $(MAKE) k8s/vald/deploy \ - HELM_VALUES=$(ROOTDIR)/.github/helm/values/values-readreplica.yaml - sleep 20 - kubectl wait --for=condition=Ready pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) - kubectl wait --for=condition=ContainersReady pod -l "app=$(AGENT_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait --for=condition=Ready pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl wait --for=condition=ContainersReady pod -l "app=$(LB_GATEWAY_IMAGE)" --timeout=$(E2E_WAIT_FOR_START_TIMEOUT) kubectl get pods diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index dfd1522718..7a9e8689d5 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -111,9 +111,9 @@ k8s/manifest/readreplica/update: \ mv $(TEMP_DIR)/vald-readreplica/templates $(ROOTDIR)/k8s/readreplica rm -rf $(TEMP_DIR) -.PHONY: k8s/vald/deploy -## deploy vald sample cluster to k8s -k8s/vald/deploy: +.PHONY: k8s/vald/manifests +## generate vald manifest +k8s/vald/manifests: helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ @@ -131,6 +131,10 @@ k8s/vald/deploy: --include-crds \ --output-dir $(TEMP_DIR) \ charts/vald + +.PHONY: k8s/vald/deploy +## deploy vald sample cluster to k8s +k8s/vald/deploy: k8s/vald/manifests @echo "Permitting error because there's some cases nothing to apply" kubectl apply -f $(TEMP_DIR)/vald/templates/manager/index || true kubectl apply -f $(TEMP_DIR)/vald/templates/agent || true @@ -146,26 +150,13 @@ k8s/vald/deploy: kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/save || true kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true kubectl apply -f $(TEMP_DIR)/vald/templates/index/operator || true - rm -rf $(TEMP_DIR) kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n" + @echo "manifest files location: $(TEMP_DIR)" + .PHONY: k8s/vald/delete ## delete vald sample cluster from k8s -k8s/vald/delete: - helm template \ - --values $(HELM_VALUES) \ - --set defaults.image.tag=$(VERSION) \ - --set agent.image.repository=$(CRORG)/$(AGENT_NGT_IMAGE) \ - --set agent.sidecar.image.repository=$(CRORG)/$(AGENT_SIDECAR_IMAGE) \ - --set discoverer.image.repository=$(CRORG)/$(DISCOVERER_IMAGE) \ - --set gateway.filter.image.repository=$(CRORG)/$(FILTER_GATEWAY_IMAGE) \ - --set gateway.lb.image.repository=$(CRORG)/$(LB_GATEWAY_IMAGE) \ - --set gateway.mirror.image.repository=$(CRORG)/$(MIRROR_GATEWAY_IMAGE) \ - --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \ - --set manager.index.operator.image.repository=$(CRORG)/$(INDEX_OPERATOR_IMAGE) \ - --include-crds \ - --output-dir $(TEMP_DIR) \ - charts/vald +k8s/vald/delete: k8s/vald/manifests kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/mirror || true kubectl delete -f $(TEMP_DIR)/vald/templates/index/operator || true kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/readreplica/rotate || true @@ -182,7 +173,6 @@ k8s/vald/delete: kubectl delete -f $(TEMP_DIR)/vald/templates/agent/ngt || true kubectl delete -f $(TEMP_DIR)/vald/templates/agent || true kubectl delete -f $(TEMP_DIR)/vald/crds || true - rm -rf $(TEMP_DIR) .PHONY: k8s/multi/vald/deploy ## deploy multiple vald sample clusters to k8s @@ -246,7 +236,7 @@ k8s/vald-helm-operator/delete: .PHONY: k8s/vald-readreplica/deploy ## deploy vald-readreplica to k8s -k8s/vald-readreplica/deploy: +k8s/vald-readreplica/deploy: k8s/vald/deploy helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ @@ -261,15 +251,26 @@ k8s/vald-readreplica/deploy: $(HELM_EXTRA_OPTIONS) \ --output-dir $(TEMP_DIR) \ charts/vald-readreplica + kubectl delete -f $(TEMP_DIR)/vald/templates/gateway || true + kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb || true + kubectl get pods + kubectl wait --for=delete pod -l app=vald-lb-gateway --timeout=600s + kubectl apply -f $(TEMP_DIR)/vald-readreplica/templates sleep 5 - # kubectl get pods - # kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=vald-readreplica --timeout=600s + kubectl get pods + kubectl wait --for=condition=ready pod -l app=vald-agent --timeout=600s + kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=vald-readreplica --timeout=600s + + kubectl apply -f $(TEMP_DIR)/vald/templates/gateway || true + kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true + + kubectl get pods .PHONY: k8s/vald-readreplica/delete ## delete vald-helm-operator from k8s -k8s/vald-readreplica/delete: +k8s/vald-readreplica/delete: k8s/vald/delete helm template \ --values $(HELM_VALUES) \ --set defaults.image.tag=$(VERSION) \ diff --git a/Makefile.d/kind.mk b/Makefile.d/kind.mk index 271acaf2cd..32fac1657f 100644 --- a/Makefile.d/kind.mk +++ b/Makefile.d/kind.mk @@ -91,8 +91,8 @@ kind/vs/start: \ && curl -fsSL https://github.com/kubernetes-csi/csi-driver-host-path/archive/refs/tags/v1.15.0.tar.gz | tar zxf - -C $(TEMP_DIR)/csi-driver-hostpath --strip-components 1 \ && cd $(TEMP_DIR)/csi-driver-hostpath \ && deploy/kubernetes-latest/deploy.sh \ - && kubectl apply -f ./examples/csi-storageclass.yaml \ - && kubectl apply -f ././examples/csi-pvc.yaml \ + && kubectl apply -f examples/csi-storageclass.yaml \ + && kubectl apply -f examples/csi-pvc.yaml \ && rm -rf $(TEMP_DIR)/csi-driver-hostpath @make k8s/metrics/metrics-server/deploy diff --git a/Makefile.d/minikube.mk b/Makefile.d/minikube.mk index ebec7bc81f..d7240770fc 100644 --- a/Makefile.d/minikube.mk +++ b/Makefile.d/minikube.mk @@ -25,9 +25,8 @@ $(BINDIR)/minikube: # Start minikube with CSI Driver and Volume Snapshots support # Only use this for development related to Volume Snapshots. Usually k3d is faster. .PHONY: minikube/start -minikube/start: \ - $(BINDIR)/docker - minikube start --driver=docker --force --alsologtostderr -v=1 +minikube/start: + minikube start minikube addons enable volumesnapshots minikube addons enable csi-hostpath-driver minikube addons disable storage-provisioner diff --git a/charts/vald/templates/_helpers.tpl b/charts/vald/templates/_helpers.tpl index 11730a90ab..f4286dee29 100644 --- a/charts/vald/templates/_helpers.tpl +++ b/charts/vald/templates/_helpers.tpl @@ -714,6 +714,10 @@ initContainers {{- $agentReadinessPort := default $.Values.defaults.server_config.healths.readiness.port $.Values.agent.server_config.healths.readiness.port }} {{- $agentReadinessPath := default $.Values.defaults.server_config.healths.readiness.readinessProbe.httpGet.path .readinessPath }} until [ "$(wget --server-response --spider --quiet http://{{ $.Values.agent.name }}.{{ $.namespace }}.svc.cluster.local:{{ $agentReadinessPort }}{{ $agentReadinessPath }} 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + {{- else if eq .target "agent-readreplica" }} + {{- $agentReadReplicaReadinessPort := default $.Values.defaults.server_config.healths.readiness.port $.Values.agent.server_config.healths.readiness.port }} + {{- $agentReadReplicaReadinessPath := default $.Values.defaults.server_config.healths.readiness.readinessProbe.httpGet.path .readinessPath }} + until [ "$(wget --server-response --spider --quiet http://{{ $.Values.agent.readreplica.name }}-0.{{ $.namespace }}.svc.cluster.local:{{ $agentReadReplicaReadinessPort }}{{ $agentReadReplicaReadinessPath }} 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do {{- else if eq .target "gateway-lb" }} {{- $lbGatewayReadinessPort := default $.Values.defaults.server_config.healths.readiness.port $.Values.gateway.lb.server_config.healths.readiness.port }} {{- $lbGatewayReadinessPath := default $.Values.defaults.server_config.healths.readiness.readinessProbe.httpGet.path .readinessPath }} From 75d10622e70fa088693ad5abc6b74fb875eedab0 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 23 Dec 2024 17:45:02 +0900 Subject: [PATCH 7/7] To disable index check in Read Replica --- .../e2e-deploy-vald-readreplica/action.yaml | 1 + .github/workflows/_detect-ci-container.yaml | 2 +- .github/workflows/e2e.yaml | 4 ++-- Makefile.d/kind.mk | 14 +++++--------- pkg/agent/core/ngt/service/ngt.go | 3 +++ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/actions/e2e-deploy-vald-readreplica/action.yaml b/.github/actions/e2e-deploy-vald-readreplica/action.yaml index eb8f4b1c79..31f7837198 100644 --- a/.github/actions/e2e-deploy-vald-readreplica/action.yaml +++ b/.github/actions/e2e-deploy-vald-readreplica/action.yaml @@ -85,6 +85,7 @@ runs: run: | make k8s/vald-readreplica/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + stern vald-discoverer & stern vald-lb-gateway & stern vald-agent-readreplica & sleep 3 diff --git a/.github/workflows/_detect-ci-container.yaml b/.github/workflows/_detect-ci-container.yaml index 59befea939..fd40bc8713 100644 --- a/.github/workflows/_detect-ci-container.yaml +++ b/.github/workflows/_detect-ci-container.yaml @@ -41,7 +41,7 @@ jobs: - name: Get Docker image tag from detection result id: get_tag_name run: | - TAG=$(echo "$TAGS" | awk '{print $1}' | awk -F '=' '{print $2}') + TAG=$(echo "$TAGS" | awk '{print $1}' | awk -F '=' '{print $2}' | sed -e 's/"//g') echo "TAG=${TAG}" echo "TAG=${TAG}" >> $GITHUB_OUTPUT env: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9edada1419..61ce1543e7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -335,12 +335,12 @@ jobs: E2E_UPDATE_COUNT=100 \ E2E_UPSERT_COUNT=100 \ E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=10m \ E2E_TARGET_POD_NAME=${POD_NAME} \ E2E_TARGET_NAMESPACE=default \ e2e/readreplica env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }} e2e-stream-crud-with-mirror: name: "E2E test (Stream CRUD) with mirror" needs: [detect-ci-container] diff --git a/Makefile.d/kind.mk b/Makefile.d/kind.mk index 32fac1657f..10885113e8 100644 --- a/Makefile.d/kind.mk +++ b/Makefile.d/kind.mk @@ -20,7 +20,7 @@ SNAPSHOTTER_VERSION=v8.2.0 ## install KinD kind/install: $(BINDIR)/kind -$(BINDIR)/kind: +$(BINDIR)/kind: $(BINDIR)/docker mkdir -p $(BINDIR) $(eval DARCH := $(subst aarch64,arm64,$(ARCH))) curl -fsSL https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(OS)-$(subst x86_64,amd64,$(shell echo $(DARCH) | tr '[:upper:]' '[:lower:]')) -o $(BINDIR)/kind @@ -28,8 +28,7 @@ $(BINDIR)/kind: .PHONY: kind/start ## start kind (kubernetes in docker) cluster -kind/start: \ - $(BINDIR)/docker +kind/start: kind create cluster --name $(NAME) @make kind/login @@ -40,8 +39,7 @@ kind/login: .PHONY: kind/stop ## stop kind (kubernetes in docker) cluster -kind/stop: \ - $(BINDIR)/docker +kind/stop: kind delete cluster --name $(NAME) .PHONY: kind/restart @@ -76,8 +74,7 @@ kind/cluster/restart: \ .PHONY: kind/vs/start ## start kind (kubernetes in docker) cluster with volume snapshot -kind/vs/start: \ - $(BINDIR)/docker +kind/vs/start: sed -e 's/apiServerAddress: "127.0.0.1"/apiServerAddress: "$(shell grep host.docker.internal /etc/hosts | cut -f1)"/' $(ROOTDIR)/k8s/debug/kind/e2e.yaml | kind create cluster --name $(NAME)-vs --config - @make kind/vs/login @@ -101,8 +98,7 @@ kind/vs/start: \ .PHONY: kind/vs/stop ## stop kind (kubernetes in docker) cluster with volume snapshot -kind/vs/stop: \ - $(BINDIR)/docker +kind/vs/stop: kind delete cluster --name $(NAME)-vs .PHONY: kind/vs/login diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index ad89d91363..eadd0d64e3 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -887,6 +887,9 @@ func (n *ngt) Start(ctx context.Context) <-chan error { if n.dcd { return nil } + if n.isReadReplica { + return nil + } n.removeInvalidIndex(ctx) ech := make(chan error, 2) n.eg.Go(safety.RecoverFunc(func() (err error) {