Merge pull request #900 from alexander-demicev/backport014

[release-0.14] Backport etcd image and RBAC

Dockerfile

@@ -38,12 +38,33 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     -o manager ${package}
 
 
+FROM ${builder_image} as etcd-snapshot-restore-builder
+WORKDIR /workspace
+
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build-arg package=./exp/etcdrestore
+ENV GOPROXY=$goproxy
+
+# Copy the sources
+COPY ./ ./
+
+# Build
+ARG ldflags
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux \
+    sh -c "cd exp/etcdrestore && ls && go build -trimpath -ldflags \"${ldflags} -extldflags '-static'\" -o manager ${package}"
+
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 LABEL org.opencontainers.image.source=https://github.com/rancher/turtles
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY --from=etcd-snapshot-restore-builder /workspace/exp/etcdrestore/manager etcd-snapshot-restore
 # Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
 USER 65532
 ENTRYPOINT ["/manager"]

Makefile

@@ -184,10 +184,6 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(ORG)/$(CONTROLLER_IMAGE_NAME)
 CONTROLLER_IMAGE_VERSION ?= $(shell git describe --abbrev=0 2>/dev/null)
 IID_FILE ?= $(shell mktemp)
 
-# etcdrestore
-ETCDRESTORE_IMAGE_NAME ?= turtles-etcd-snapshot-restore
-ETCDRESTORE_IMG ?= $(REGISTRY)/$(ORG)/$(ETCDRESTORE_IMAGE_NAME)
-
 # Release
 # Exclude tags with the prefix 'test/'
 RELEASE_TAG ?= $(shell git describe --abbrev=0 --exclude 'test/*' 2>/dev/null)
@@ -388,35 +384,6 @@ docker-build-and-push: buildx-machine docker-pull-prerequisites ## Run docker-bu
 			--build-arg package=. \
 			--build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG):$(TAG)
 
-## --------------------------------------
-## Docker - etcdrestore
-## --------------------------------------
-
-.PHONY: docker-build-etcdrestore ## Build the docker image for etcdrestore
-docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
-	## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
-	# buildx does not support using local registry for multi-architecture images
-	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
-			--platform $(ARCH) \
-			--load \
-			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
-			--build-arg goproxy=$(GOPROXY) \
-			--build-arg package=./exp/etcdrestore \
-			--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain
-
-.PHONY: docker-build-and-push-etcdrestore
-docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures
-	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
-			--platform $(TARGET_PLATFORMS) \
-			--push \
-			--sbom=true \
-			--attest type=provenance,mode=max \
-			--iidfile=$(IID_FILE) \
-			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
-			--build-arg goproxy=$(GOPROXY) \
-			--build-arg package=./exp/etcdrestore \
-			--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain
-
 docker-list-all:
 	@echo $(CONTROLLER_IMG):${TAG}
 
@@ -585,12 +552,12 @@ build-chart: $(HELM) $(KUSTOMIZE) $(RELEASE_DIR) $(CHART_RELEASE_DIR) $(CHART_PA
 	$(KUSTOMIZE) build ./exp/etcdrestore/config/default > $(CHART_DIR)/templates/rancher-turtles-exp-etcdrestore-components.yaml
 	./scripts/process-exp-etcdrestore-manifests.sh $(CHART_DIR)/templates/rancher-turtles-exp-etcdrestore-components.yaml
 	cp -rf $(CHART_DIR)/* $(CHART_RELEASE_DIR)
-	
+
 	sed -i'' -e 's@image: .*@image: '"$(CONTROLLER_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@imageVersion: .*@imageVersion: '"$(RELEASE_TAG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(CHART_RELEASE_DIR)/values.yaml
 
-	sed -i'' -e '/etcd-snapshot-restore:/,/image:/ s@image: .*@image: '"$(ETCDRESTORE_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
+	sed -i'' -e '/etcd-snapshot-restore:/,/image:/ s@image: .*@image: '"$(CONTROLLER_IMG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e '/etcd-snapshot-restore:/,/imageVersion:/ s@imageVersion: .*@imageVersion: '"$(RELEASE_TAG)"'@' $(CHART_RELEASE_DIR)/values.yaml
 	sed -i'' -e '/etcd-snapshot-restore:/,/imagePullPolicy:/ s@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' $(CHART_RELEASE_DIR)/values.yaml
 
@@ -627,17 +594,14 @@ test-e2e-push-image: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image-push
 .PHONY: e2e-image
 e2e-image: $(CACHE_DIR) ## Build the image for e2e tests
 	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build
-	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 ETCDRESTORE_IMAGE_NAME=turtles-etcd-snapshot-restore-e2e $(MAKE) docker-build-etcdrestore
-	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e ETCDRESTORE_IMG=$(REGISTRY)/$(ORG)/turtles-etcd-snapshot-restore-e2e \
+	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e \
 	CONTROLLER_IMAGE_VERSION=v0.0.1 \
 	$(MAKE) build-chart
 
 .PHONY: e2e-image-push
 e2e-image-push: $(CACHE_DIR) ## Push the image for e2e tests
 	TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e \
 	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push
-	TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 ETCDRESTORE_IMAGE_NAME=turtles-etcd-snapshot-restore-e2e \
-	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push-etcdrestore
 
 .PHONY: compile-e2e
 e2e-compile: ## Test e2e compilation

Tiltfile

@@ -35,7 +35,7 @@ if settings.get("trigger_mode") == "manual":
 if settings.get("default_registry") != "":
     default_registry(settings.get("default_registry"))
 
-always_enable_projects = ["turtles", "turtles-etcdsnapshotrestore", "turtles-capiproviders"]
+always_enable_projects = ["turtles", "turtles-capiproviders"]
 
 projects = {
     "turtles": {
@@ -51,19 +51,6 @@ projects = {
         "kustomize_dir": "config/default",
         "label": "turtles"
     },
-    "turtles-etcdsnapshotrestore": {
-        "context": "exp/etcdrestore",
-        "image": "ghcr.io/rancher/turtles-etcd-snapshot-restore:dev",
-        "live_reload_deps": [
-            "main.go",
-            "go.mod",
-            "go.sum",
-            "controllers",
-            "webhooks",
-        ],
-        "kustomize_dir": "config/default",
-        "label": "turtles-etcdsnapshotrestore"
-    },
     "turtles-capiproviders": {
         "context": ".",
         "live_reload_deps": [

charts/rancher-turtles/templates/rancher-turtles-exp-etcdrestore-components.yaml

@@ -657,7 +657,7 @@ spec:
       - args:
         - --leader-elect
         command:
-        - /manager
+        - ./etcd-snapshot-restore
         env:
         - name: POD_NAMESPACE
           valueFrom:

exp/etcdrestore/config/default/manager_image_patch.yaml

@@ -8,4 +8,5 @@ spec:
     spec:
       containers:
         - image: ghcr.io/rancher/turtles-etcd-snapshot-restore:dev
-          name: manager
+          name: manager
+          command: ["./etcd-snapshot-restore"]

scripts/turtles-dev.sh

@@ -22,7 +22,7 @@ fi
 
 RANCHER_VERSION=${RANCHER_VERSION:-v2.10.0}
 CLUSTER_NAME=${CLUSTER_NAME:-capi-test}
-ETCD_CONTROLLER_IMAGE=${ETCD_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles-etcd-snapshot-restore}
+ETCD_CONTROLLER_IMAGE=${ETCD_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles}
 ETCD_CONTROLLER_IMAGE_TAG=${ETCD_CONTROLLER_IMAGE_TAG:-dev}
 USE_TILT_DEV=${USE_TILT_DEV:-true}
 
@@ -73,7 +73,7 @@ install_local_rancher_turtles_chart() {
 	# Build the chart locally
 	make build-chart
 	# Build the etcdrestore controller image
-	make docker-build-etcdrestore
+	make docker-build
 	# Load the etcdrestore controller image into the kind cluster
 	kind load docker-image $ETCD_CONTROLLER_IMAGE:$ETCD_CONTROLLER_IMAGE_TAG --name $CLUSTER_NAME
 	# Install the Rancher Turtles using a local chart with 'etcd-snapshot-restore' feature flag enabled
@@ -83,6 +83,8 @@ install_local_rancher_turtles_chart() {
 		--set cluster-api-operator.enabled=true \
 		--set cluster-api-operator.cluster-api.enabled=false \
 		--set rancherTurtles.features.etcd-snapshot-restore.enabled=true \
+		--set rancherTurtles.features.etcd-snapshot-restore.imageVersion=dev \
+		--set rancherTurtles.imageVersion=dev \
 		--dependency-update \
 		--create-namespace --wait \
 		--timeout 180s

test/e2e/config/operator.yaml

@@ -4,8 +4,6 @@ images:
 # Use local dev images built source tree;
 - name: ghcr.io/rancher/turtles-e2e:v0.0.1
   loadBehavior: tryLoad
-- name: ghcr.io/rancher/turtles-etcd-snapshot-restore-e2e:v0.0.1
-  loadBehavior: tryLoad
 
 intervals:
   default/wait-controllers: ["15m", "10s"]

test/e2e/specs/import_gitops_mgmtv3.go

@@ -44,6 +44,7 @@ import (
 	"github.com/rancher/turtles/test/e2e"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	turtlesannotations "github.com/rancher/turtles/util/annotations"
 )
 
 type CreateMgmtV3UsingGitOpsSpecInput struct {
@@ -137,13 +138,12 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
 		}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
 		By("Waiting for the rancher cluster to be ready")
 
-		// TODO: re-enable the check after rancher 2.10 update and verified existence of the annotation
-		// By("Rancher cluster should have the 'NoCreatorRBAC' annotation")
-		// Eventually(func() bool {
-		// 	Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
-		// 	_, found := rancherCluster.Annotations[turtlesannotations.NoCreatorRBACAnnotation]
-		// 	return found
-		// }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
+		By("Rancher cluster should have the 'NoCreatorRBAC' annotation")
+		Eventually(func() bool {
+			Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
+			_, found := rancherCluster.Annotations[turtlesannotations.NoCreatorRBACAnnotation]
+			return found
+		}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
 
 		By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
 		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{

test/e2e/specs/migrate_gitops_provv1_mgmtv3.go

@@ -46,6 +46,7 @@ import (
 	"github.com/rancher/turtles/test/e2e"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	turtlesannotations "github.com/rancher/turtles/util/annotations"
 	turtlesnaming "github.com/rancher/turtles/util/naming"
 )
 
@@ -176,13 +177,12 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 			return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionReady)
 		}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
 
-		// TODO: re-enable the check after rancher 2.10 update and verified existence of the annotation
-		// By("Rancher cluster should have the 'NoCreatorRBAC' annotation")
-		// Eventually(func() bool {
-		// 	Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
-		// 	_, found := rancherCluster.Annotations[turtlesannotations.NoCreatorRBACAnnotation]
-		// 	return found
-		// }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
+		By("Rancher cluster should have the 'NoCreatorRBAC' annotation")
+		Eventually(func() bool {
+			Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
+			_, found := rancherCluster.Annotations[turtlesannotations.NoCreatorRBACAnnotation]
+			return found
+		}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
 
 		By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
 		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{