✨ test/e2e: use vSphere projects from Boskos

.dockerignore

@@ -5,7 +5,6 @@
 .golangci.yml
 bin/
 **/*.yaml
-hack/
 out/
 docs/
 packaging/

Makefile

@@ -77,7 +77,7 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
 E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
-E2E_IPAM_KUBECONFIG ?=
+E2E_VSPHERE_IP_POOL ?=
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
 E2E_GOVMOMI_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-govmomi
 E2E_SUPERVISOR_TEMPLATE_DIR := $(E2E_TEMPLATE_DIR)/infrastructure-vsphere-supervisor
@@ -186,6 +186,9 @@ IMPORT_BOSS_PKG := k8s.io/code-generator/cmd/import-boss
 
 CAPI_HACK_TOOLS_VER := ef04465b2ba76214eea570e27e8146c96412e32a # Note: this is the commit ID of CAPI v1.7.1
 
+BOSKOSCTL_BIN := boskosctl
+BOSKOSCTL := $(abspath $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN))
+
 CONVERSION_VERIFIER_VER := $(CAPI_HACK_TOOLS_VER)
 CONVERSION_VERIFIER_BIN := conversion-verifier
 CONVERSION_VERIFIER := $(abspath $(TOOLS_BIN_DIR)/$(CONVERSION_VERIFIER_BIN)-$(CONVERSION_VERIFIER_VER))
@@ -230,6 +233,14 @@ VM_OPERATOR_ALL_ARCH = amd64 arm64
 NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator
 NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME)
 
+# boskosctl
+BOSKOSCTL_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/boskosctl
+BOSKOSCTL_IMG_TAG ?= $(shell git describe --always --dirty)
+
+# openvpn
+OPENVPN_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/openvpn
+OPENVPN_IMG_TAG ?= $(shell git describe --always --dirty)
+
 # It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971
 
 TAG ?= dev
@@ -560,6 +571,28 @@ docker-build-net-operator: docker-pull-prerequisites ## Build the docker image f
 		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \
 	fi
 
+.PHONY: docker-build-boskosctl
+docker-build-boskosctl:
+	cat hack/tools/boskosctl/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) --file -
+	docker tag $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) $(BOSKOSCTL_IMG):latest
+.PHONY: build
+
+.PHONY: docker-push-boskosctl
+docker-push-boskosctl:
+	docker push $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG)
+	docker push $(BOSKOSCTL_IMG):latest
+
+.PHONY: docker-build-openvpn
+docker-build-openvpn:
+	cat hack/tools/openvpn/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(OPENVPN_IMG):$(OPENVPN_IMG_TAG) --file -
+	docker tag $(OPENVPN_IMG):$(OPENVPN_IMG_TAG) $(OPENVPN_IMG):latest
+.PHONY: build
+
+.PHONY: docker-push-openvpn
+docker-push-openvpn:
+	docker push $(OPENVPN_IMG):$(OPENVPN_IMG_TAG)
+	docker push $(OPENVPN_IMG):latest
+
 ## --------------------------------------
 ## Testing
 ## --------------------------------------
@@ -621,7 +654,7 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
-		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
+		--e2e.ip-pool='$(E2E_VSPHERE_IP_POOL)'
 
 ## --------------------------------------
 ## Release
@@ -952,6 +985,9 @@ $(CONVERSION_GEN_BIN): $(CONVERSION_GEN) ## Build a local copy of conversion-gen
 .PHONY: $(PROWJOB_GEN_BIN)
 $(PROWJOB_GEN_BIN): $(PROWJOB_GEN) ## Build a local copy of prowjob-gen.
 
+.PHONY: $(BOSKOSCTL_BIN)
+$(BOSKOSCTL_BIN): $(BOSKOSCTL) ## Build a local copy of boskosctl.
+
 .PHONY: $(CONVERSION_VERIFIER_BIN)
 $(CONVERSION_VERIFIER_BIN): $(CONVERSION_VERIFIER) ## Build a local copy of conversion-verifier.
 
@@ -1006,6 +1042,9 @@ $(CONTROLLER_GEN): # Build controller-gen.
 $(CONVERSION_GEN): # Build conversion-gen.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONVERSION_GEN_PKG) $(CONVERSION_GEN_BIN) $(CONVERSION_GEN_VER)
 
+$(BOSKOSCTL): # Build boskosctl from tools folder.
+	go build -o $(TOOLS_BIN_DIR)/$(BOSKOSCTL_BIN) ./hack/tools/boskosctl
+
 $(CONVERSION_VERIFIER): # Build conversion-verifier.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_TOOLS_BUILD) $(CONVERSION_VERIFIER_PKG) $(CONVERSION_VERIFIER_BIN) $(CONVERSION_VERIFIER_VER)
 

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20240404200847-de75746a9505
 	github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20240404200847-de75746a9505
 	github.com/vmware/govmomi v0.37.3
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/mod v0.18.0
 	golang.org/x/tools v0.22.0

go.sum

@@ -625,6 +625,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

hack/e2e.sh

@@ -24,7 +24,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
 RE_VCSIM='\[vcsim\\]'
 
 # In CI, ARTIFACTS is set to a different directory. This stores the value of
-# ARTIFACTS i1n ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
+# ARTIFACTS in ORIGINAL_ARTIFACTS and replaces ARTIFACTS by a temporary directory
 # which gets cleaned up from credentials at the end of the test.
 export ORIGINAL_ARTIFACTS=""
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
@@ -36,10 +36,22 @@ fi
 # shellcheck source=./hack/ensure-go.sh
 source "${REPO_ROOT}/hack/ensure-go.sh"
 
-# shellcheck source=./hack/ensure-kubectl.sh
-source "${REPO_ROOT}/hack/ensure-kubectl.sh"
+export BOSKOS_RESOURCE_OWNER=cluster-api-provider-vsphere
+if [[ "${JOB_NAME}" != "" ]]; then
+  export BOSKOS_RESOURCE_OWNER="${JOB_NAME}/${BUILD_ID}"
+fi
+export BOSKOS_RESOURCE_TYPE=vsphere-project-cluster-api-provider
 
 on_exit() {
+  # Only handle Boskos when we have to (not for vcsim)
+  if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
+    # Stop boskos heartbeat
+    [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}"
+
+    # If Boskos is being used then release the vsphere project.
+    [ -z "${BOSKOS_HOST:-}" ] || docker run -e VSPHERE_USERNAME -e VSPHERE_PASSWORD gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest release --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" --vsphere-server="${VSPHERE_SERVER}" --vsphere-tls-thumbprint="${VSPHERE_TLS_THUMBPRINT}" --vsphere-folder="${BOSKOS_RESOURCE_FOLDER}" --vsphere-resource-pool="${BOSKOS_RESOURCE_POOL}"
+  fi
+
   # kill the VPN only when we started it (not vcsim)
   if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     docker kill vpn
@@ -86,18 +98,13 @@ export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml"
 export E2E_CONF_OVERRIDE_FILE=""
 export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.6-0-gde75746a}"
-export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
 export GC_KIND="false"
 
 # Make tests run in-parallel
 export GINKGO_NODES=5
 
-# Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
-# for kube-vip.
-export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
-
-# Only run the vpn/check for IPAM when we need them (not vcsim)
+# Only run the vpn/check for IPAM when we need them (not for vcsim)
 if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Run the vpn client in container
   docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
@@ -107,11 +114,11 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
   # Tail the vpn logs
   docker logs vpn
 
-  # Wait until the VPN connection is active and we are able to reach the ipam cluster
-  function wait_for_ipam_reachable() {
+  # Wait until the VPN connection is active.
+  function wait_for_vpn_up() {
     local n=0
     until [ $n -ge 30 ]; do
-      kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
+      curl "https://${VSPHERE_SERVER}" --connect-timeout 2 -k && RET=$? || RET=$?
       if [[ "$RET" -eq 0 ]]; then
         break
       fi
@@ -120,7 +127,39 @@ if [[ ! "${GINKGO_FOCUS:-}" =~ $RE_VCSIM ]]; then
     done
     return "$RET"
   }
-  wait_for_ipam_reachable
+  wait_for_vpn_up
+
+  # If BOSKOS_HOST is set then acquire a vsphere-project from Boskos.
+  if [ -n "${BOSKOS_HOST:-}" ]; then
+    # Check out the account from Boskos and store the produced environment
+    # variables in a temporary file.
+    account_env_var_file="$(mktemp)"
+    docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest acquire --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-type="${BOSKOS_RESOURCE_TYPE}" 1>"${account_env_var_file}"
+    checkout_account_status="${?}"
+
+    # If the checkout process was a success then load the account's
+    # environment variables into this process.
+    # shellcheck disable=SC1090
+    [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}"
+    export BOSKOS_RESOURCE_NAME=${BOSKOS_RESOURCE_NAME}
+    export VSPHERE_FOLDER=${BOSKOS_RESOURCE_FOLDER}
+    export VSPHERE_RESOURCE_POOL=${BOSKOS_RESOURCE_POOL}
+    export E2E_VSPHERE_IP_POOL="${BOSKOS_RESOURCE_IP_POOL}"
+
+    # Always remove the account environment variable file. It contains
+    # sensitive information.
+    rm -f "${account_env_var_file}"
+
+    if [ ! "${checkout_account_status}" = "0" ]; then
+      echo "error getting vsphere project from Boskos" 1>&2
+      exit "${checkout_account_status}"
+    fi
+
+    # Run the heartbeat to tell boskos periodically that we are still
+    # using the checked out account.
+    docker run gcr.io/k8s-staging-capi-vsphere/extra/boskosctl:latest heartbeat --boskos-host="${BOSKOS_HOST}" --resource-owner="${BOSKOS_RESOURCE_OWNER}" --resource-name="${BOSKOS_RESOURCE_NAME}" >>"${ARTIFACTS}/boskos-heartbeat.log" 2>&1 &
+    HEART_BEAT_PID=$!
+  fi
 fi
 
 make envsubst

hack/tools/boskosctl/Dockerfile

@@ -0,0 +1,71 @@
+# syntax=docker/dockerfile:1.4
+
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the manager binary
+# Run this with docker build --build-arg builder_image=<golang:x.y.z>
+ARG builder_image
+
+# Build architecture
+ARG ARCH
+
+# Ignore Hadolint rule "Always tag the version of an image explicitly."
+# It's an invalid finding since the image is explicitly set in the Makefile.
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# hadolint ignore=DL3006
+FROM ${builder_image} as builder
+WORKDIR /workspace
+
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+# Run this with docker build --build-arg package=./controlplane/kubeadm or --build-arg package=./bootstrap/kubeadm
+ENV GOPROXY=$goproxy
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# Copy the sources
+COPY ./ ./
+
+# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go build .
+
+# Build
+ARG ldflags
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN ls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
+    -o boskosctl ./hack/tools/boskosctl
+
+# Production image
+FROM gcr.io/distroless/static:nonroot-amd64
+
+WORKDIR /
+COPY --from=builder /workspace/boskosctl .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
+USER 65532
+ENTRYPOINT ["/boskosctl"]