Add ownerRef check for vSphereClusterIdentity

Signed-off-by: killianmuldoon <kmuldoon@vmware.com>

Makefile

@@ -315,6 +315,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/pci > $(E2E_TEMPLATE_DIR)/main/cluster-template-pci.yaml
 	# for DHCP overrides
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/dhcp-overrides > $(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/main/failuredomains > $(E2E_TEMPLATE_DIR)/main/cluster-template-failuredomains.yaml
 
 
 ## --------------------------------------

test/e2e/config/vsphere-ci.yaml

@@ -92,6 +92,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-storage-policy.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-topology.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-failuredomains.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml"
           - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml"

test/e2e/config/vsphere-dev.yaml

@@ -97,6 +97,7 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/clusterclass-quick-start.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-failuredomains.yaml"
           - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml"
 
 variables:

test/e2e/data/infrastructure-vsphere/main/failuredomains/cluster-identity.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereClusterIdentity
+metadata:
+  name: "${CLUSTER_NAME}"
+  namespace: "${NAMESPACE}"
+spec:
+  secretName: identity
+  allowedNamespaces:
+    selector:
+      matchLabels:
+        kubernetes.io/metadata.name: '${NAMESPACE}'

test/e2e/data/infrastructure-vsphere/main/failuredomains/drop-existing-secret.yaml

@@ -0,0 +1,7 @@
+# This secret is not needed. This cluster uses a ClusterIdentity instead
+$patch: delete
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}

test/e2e/data/infrastructure-vsphere/main/failuredomains/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+  - cluster-identity.yaml
+patchesStrategicMerge:
+  - ../commons/cluster-resource-set-label.yaml
+  - ../commons/cluster-network-CIDR.yaml
+  - ../commons/cluster-resource-set-csi-insecure.yaml
+  - vsphereclusteridentity.yaml
+  - drop-existing-secret.yaml
+#  - ../commons/failure-domain.yaml

test/e2e/data/infrastructure-vsphere/main/failuredomains/vsphereclusteridentity.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereCluster
+metadata:
+  name: "${CLUSTER_NAME}"
+  namespace: "${NAMESPACE}"
+spec:
+  identityRef:
+    kind: VSphereClusterIdentity
+    name: "${CLUSTER_NAME}"
+

test/e2e/ownerreference_helpers.go → test/e2e/owner_reference_test.go

@@ -17,17 +17,71 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/pointer"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/yaml"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 )
 
+var _ = Describe("OwnerReference checks with FailureDomains and ClusterIdentity [PR-Blocking]", func() {
+	// Before running the test create the secret used by the VSphereClusterIdentity to connect to the vCenter.
+	BeforeEach(func() {
+		createVsphereIdentitySecret(ctx, bootstrapClusterProxy)
+	})
+
+	capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {
+		return capi_e2e.QuickStartSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			Flavor:                pointer.String("failuredomains"),
+			PostMachinesProvisioned: func(proxy framework.ClusterProxy, namespace, clusterName string) {
+				// This check ensures that owner references are resilient - i.e. correctly re-reconciled - when removed.
+				framework.ValidateOwnerReferencesResilience(ctx, proxy, namespace, clusterName,
+					framework.CoreOwnerReferenceAssertion,
+					framework.KubeadmBootstrapOwnerReferenceAssertions,
+					framework.KubeadmControlPlaneOwnerReferenceAssertions,
+					framework.ExpOwnerReferenceAssertions,
+					VSphereKubernetesReferenceAssertions,
+					VSphereReferenceAssertions,
+				)
+				// This check ensures that owner references are always updated to the most recent apiVersion.
+				framework.ValidateOwnerReferencesOnUpdate(ctx, proxy, namespace, clusterName,
+					framework.CoreOwnerReferenceAssertion,
+					framework.KubeadmBootstrapOwnerReferenceAssertions,
+					framework.KubeadmControlPlaneOwnerReferenceAssertions,
+					framework.ExpOwnerReferenceAssertions,
+					VSphereKubernetesReferenceAssertions,
+					VSphereReferenceAssertions,
+				)
+			},
+		}
+	})
+
+	// We must delete the secret created for the VSphereClusterIdentity and then delete it.
+	// TODO: The ClusterIdentity is not correctly cleaned up.
+	AfterEach(func() {
+		deleteVsphereIdentitySecret(ctx, bootstrapClusterProxy)
+	})
+
+})
+
 var (
 	VSphereKubernetesReferenceAssertions = map[string]func([]metav1.OwnerReference) error{
 		// Need custom Kubernetes assertions for secrets. Secrets in the CAPV tests can also be owned by the vSphereCluster.
@@ -41,6 +95,7 @@ var (
 				[]metav1.OwnerReference{clusterResourceSetOwner},
 				// Secrets created as an identityReference for a vSphereCluster should be owned but the vSphereCluster.
 				[]metav1.OwnerReference{vSphereClusterOwner},
+				// TODO: Not capturing a secret linked to a vSphereClusterIdentity here.
 			)
 		},
 		"ConfigMap": func(owners []metav1.OwnerReference) error {
@@ -69,9 +124,12 @@ var (
 		"VSphereVM": func(owners []metav1.OwnerReference) error {
 			return framework.HasExactOwners(owners, vSphereMachineOwner)
 		},
-		"VSphereClusterIdentity": func(owners []metav1.OwnerReference) error { panic("IMPLEMENT ME") },
-		"VSphereFailureDomain":   func(owners []metav1.OwnerReference) error { panic("IMPLEMENT ME") },
-		"vSphereDeploymentZone":  func(owners []metav1.OwnerReference) error { panic("IMPLEMENT ME") },
+		// VSphereClusterIdentity does not have any owners.
+		"VSphereClusterIdentity": func(owners []metav1.OwnerReference) error {
+			return framework.HasExactOwners(owners)
+		},
+		"VSphereFailureDomain":  func(owners []metav1.OwnerReference) error { panic("IMPLEMENT ME") },
+		"vSphereDeploymentZone": func(owners []metav1.OwnerReference) error { panic("IMPLEMENT ME") },
 	}
 )
 
@@ -93,3 +151,42 @@ var (
 	// CAPBK owner.
 	kubeadmConfigController = metav1.OwnerReference{Kind: "KubeadmConfig", APIVersion: bootstrapv1.GroupVersion.String(), Controller: pointer.Bool(true)}
 )
+
+func deleteVsphereIdentitySecret(ctx context.Context, bootstrapClusterProxy framework.ClusterProxy) bool {
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "capv-system",
+			Name:      "identity",
+		},
+	}
+	Expect(bootstrapClusterProxy.GetClient().Get(ctx, ctrlclient.ObjectKey{Namespace: s.Namespace, Name: s.Name}, s)).To(Succeed())
+	ctrlutil.RemoveFinalizer(s, infrav1.SecretIdentitySetFinalizer)
+	Expect(bootstrapClusterProxy.GetClient().Update(ctx, s)).To(Succeed())
+	Expect(bootstrapClusterProxy.GetClient().Delete(ctx,
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "capv-system",
+				Name:      "identity",
+			},
+		})).To(Succeed())
+	return true
+}
+func createVsphereIdentitySecret(ctx context.Context, bootstrapClusterProxy framework.ClusterProxy) bool {
+	credentialsSecret := &corev1.Secret{}
+	Expect(bootstrapClusterProxy.GetClient().Get(ctx, ctrlclient.ObjectKey{Namespace: "capv-system", Name: "capv-manager-bootstrap-credentials"}, credentialsSecret)).To(Succeed())
+	data := credentialsSecret.Data["credentials.yaml"]
+	credentials := map[string]string{}
+	Expect(yaml.Unmarshal(data, &credentials)).To(Succeed())
+	Expect(bootstrapClusterProxy.GetClient().Create(ctx,
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "capv-system",
+				Name:      "identity",
+			},
+			Data: map[string][]byte{
+				"password": []byte(credentials["password"]),
+				"username": []byte(credentials["username"]),
+			},
+		})).To(Succeed())
+	return true
+}