hack: add capv-janitor for automated ci cleanup

Makefile

@@ -240,6 +240,8 @@ RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 VERSION ?= $(shell cat clusterctl-settings.json | jq .config.nextVersion -r)
 OVERRIDES_DIR := $(HOME)/.cluster-api/overrides/infrastructure-vsphere/$(VERSION)
 
+JANITOR_DIR ?= ./$(TOOLS_DIR)/janitor
+
 help:  # Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[0-9A-Za-z_-]+:.*?##/ { printf "  \033[36m%-50s\033[0m %s\n", $$1, $$2 } /^\$$\([0-9A-Za-z_-]+\):.*?##/ { gsub("_","-", $$1); printf "  \033[36m%-50s\033[0m %s\n", tolower(substr($$1, 3, length($$1)-7)), $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
@@ -277,6 +279,10 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 		paths=github.com/vmware-tanzu/vm-operator/api/v1alpha1/... \
 		crd:crdVersions=v1 \
 		output:crd:dir=$(VMOP_CRD_ROOT)
+	$(CONTROLLER_GEN) \
+		paths=$(JANITOR_DIR) \
+		output:rbac:dir=$(JANITOR_DIR)/config/rbac \
+		rbac:roleName=janitor-role
 
 .PHONY: generate-go-deepcopy
 generate-go-deepcopy: $(CONTROLLER_GEN) ## Generate deepcopy go code for core

hack/tools/janitor/Dockerfile

@@ -0,0 +1,56 @@
+# syntax=docker/dockerfile:1.4
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the manager binary
+ARG GOLANG_VERSION=golang:1.20.12
+FROM --platform=${BUILDPLATFORM} ${GOLANG_VERSION} as builder
+WORKDIR /workspace
+
+# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy
+ARG goproxy=https://proxy.golang.org
+ENV GOPROXY=${goproxy}
+
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+# Copy the sources
+COPY ./ ./
+
+# Build
+ARG TARGETOS
+ARG TARGETARCH
+ARG ldflags
+WORKDIR /workspace
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+    go build -a -ldflags "${ldflags} -extldflags '-static'" \
+    -o /out/capv-janitor ./hack/tools/janitor
+
+# Copy the capv-janitor into a thin image
+ARG TARGETPLATFORM
+FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /out/capv-janitor .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying PSPs
+USER 65532
+ENTRYPOINT ["/capv-janitor"]

hack/tools/janitor/Makefile

@@ -0,0 +1,29 @@
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all: build
+
+VERSION ?= $(shell git describe --always --dirty)
+IMAGE_NAME ?= gcr.io/k8s-staging-capi-vsphere/extra/capv-janitor
+IMAGE_TAG ?= $(IMAGE_NAME):$(VERSION)
+
+build:
+	docker build -t $(IMAGE_TAG) -f Dockerfile ../../..
+	docker tag $(IMAGE_TAG) $(IMAGE_NAME):latest
+.PHONY: build
+
+push:
+	docker push $(IMAGE_TAG)
+	docker push $(IMAGE_NAME):latest
+.PHONY: push

hack/tools/janitor/README.md

@@ -0,0 +1,32 @@
+# janitor
+
+The janitor is a tool for CI to cleanup objects leftover from failed or killed prowjobs.
+It can be run regularily as cronjob.
+
+It tries to delete:
+
+* vSphere: virtual machines in the configured folders which exist longer than the configured `--max-age` flag.
+* vSphere: cluster modules which do not refer any virtual machine
+* IPAM: IPAddressClaims which exist longer than the configured `--max-age` flag
+
+## Deployment
+
+1. (Optional:) Build and push image:
+
+  ```sh
+  cd hack/tools/janitor
+  make build push
+  ```
+
+2. Deploy using kustomize and envsubst:
+
+  ```sh
+  export 
+  export VSPHERE_SERVER=""
+  export VSPHERE_USERNAME=""
+  export VSPHERE_PASSWORD=""
+  export VSPHERE_TLS_THUMBPRINT=""
+
+  cd hack/tools/janitor
+  kustomize build config/default | envsubst  | kubectl apply -f -
+  ```

hack/tools/janitor/config/cronjob/credentials.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: janitor
+  namespace: default
+type: Opaque
+stringData:
+  VSPHERE_USERNAME: "${VSPHERE_USERNAME}"
+  VSPHERE_PASSWORD: "${VSPHERE_PASSWORD}"
+  VSPHERE_SERVER: "${VSPHERE_SERVER}"
+  VSPHERE_TLS_THUMBPRINT: "${VSPHERE_TLS_THUMBPRINT}"

hack/tools/janitor/config/cronjob/cronjob.yaml

@@ -0,0 +1,47 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: janitor
+  namespace: default
+spec:
+  # Run twice a day
+  schedule: "0 */12 * * *"
+  concurrencyPolicy: Replace
+  failedJobsHistoryLimit: 3
+  successfulJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - args:
+            - --dry-run=false
+            - --min-age=12h
+            # In CAPV's CI IPAddressClaims are created in the default namespace.
+            - --ipam-namespace=default
+            # The directories used in CI to cleanup.
+            - --folder=/SDDC-Datacenter/host/Cluster-1/Resources/Compute-ResourcePool/cluster-api-provider-vsphere
+            - --folder=/SDDC-Datacenter/host/Cluster-1/Resources/Compute-ResourcePool/cloud-provider-vsphere
+            - --folder=/SDDC-Datacenter/host/Cluster-1/Resources/Compute-ResourcePool/image-builder
+            - --folder=/dc0/vm/folder0/cluster-api-provider-vsphere
+            image: gcr.io/k8s-staging-capi-vsphere/extra/capv-janitor:latest
+            imagePullPolicy: IfNotPresent
+            name: capv-janitor
+            envFrom:
+            - secretRef:
+                name: capv-janitor
+                optional: false
+            securityContext:
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop:
+                - ALL
+              runAsUser: 65532
+              runAsGroup: 65532
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          serviceAccountName: janitor
+          terminationGracePeriodSeconds: 10

hack/tools/janitor/config/cronjob/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: capv-janitor
+
+resources:
+- credentials.yaml
+- cronjob.yaml
+- namespace.yaml
+- service_account.yaml

hack/tools/janitor/config/cronjob/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: janitor

hack/tools/janitor/config/cronjob/service_account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: janitor
+  namespace: janitor

hack/tools/janitor/config/default/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: capv-
+
+commonLabels:
+  component: capv-janitor
+
+bases:
+- ../cronjob
+- ../rbac

hack/tools/janitor/config/rbac/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# In CAPV's CI IPAddressClaims are created in the default namespace.
+namespace: default
+
+resources:
+- role.yaml
+- rolebinding.yaml

hack/tools/janitor/config/rbac/role.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: janitor-role
+rules:
+- apiGroups:
+  - ipam.cluster.x-k8s.io
+  resources:
+  - ipaddressclaims
+  verbs:
+  - delete
+  - get
+  - list
+  - watch

hack/tools/janitor/config/rbac/rolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: janitor-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: janitor-role
+subjects:
+- kind: ServiceAccount
+  name: janitor
+  namespace: capv-janitor