tmp test local finalizer assertion

test/e2e/ownerrefs_finalizers_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -28,8 +29,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
@@ -101,7 +104,7 @@ var _ = Describe("Ensure OwnerReferences and Finalizers are resilient with Failu
 					)
 					// This check ensures that finalizers are resilient - i.e. correctly re-reconciled, when removed.
 					By("Checking that finalizers are resilient")
-					framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName),
+					ValidateFinalizersResilience(ctx, proxy, namespace, clusterName, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName),
 						framework.CoreFinalizersAssertionWithLegacyClusters,
 						framework.KubeadmControlPlaneFinalizersAssertion,
 						framework.ExpFinalizersAssertion,
@@ -339,3 +342,171 @@ func forcePeriodicReconcile(ctx context.Context, c ctrlclient.Client, namespace
 		}
 	}()
 }
+
+// ------ TMP fork to test things in CAPI quickly
+
+// ValidateFinalizersResilience checks that expected finalizers are in place, deletes them, and verifies that expected finalizers are properly added again.
+func ValidateFinalizersResilience(ctx context.Context, proxy framework.ClusterProxy, namespace, clusterName string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction, finalizerAssertions ...map[string]func(name types.NamespacedName) []string) {
+	clusterKey := ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName}
+	allFinalizerAssertions, err := concatenateFinalizerAssertions(finalizerAssertions...)
+	Expect(err).ToNot(HaveOccurred())
+
+	// Collect all objects where finalizers were initially set
+	byf("Check that the finalizers are as expected")
+	_, err = getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction)
+	Expect(err).ToNot(HaveOccurred(), "Finalizers are not as expected")
+
+	byf("Removing all the finalizers")
+	// Setting the paused property on the Cluster resource will pause reconciliations, thereby having no effect on Finalizers.
+	// This also makes debugging easier.
+	setClusterPause(ctx, proxy.GetClient(), clusterKey, true)
+
+	// We are testing the worst-case scenario, i.e. all finalizers are deleted.
+	// Once all Clusters are paused remove all the Finalizers from all objects in the graph.
+	// The reconciliation loop should be able to recover from this, by adding the required Finalizers back.
+	removeFinalizers(ctx, proxy, namespace, ownerGraphFilterFunction)
+
+	// Unpause the cluster.
+	setClusterPause(ctx, proxy.GetClient(), clusterKey, false)
+
+	// Check that the Finalizers are as expected after further reconciliations.
+	byf("Check that the finalizers are rebuilt as expected")
+	// assertFinalizersExist(ctx, proxy, namespace, objectsWithFinalizers, allFinalizerAssertions, ownerGraphFilterFunction)
+	Eventually(func() error {
+		_, err := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction)
+
+		return err
+	}).WithTimeout(1*time.Minute).WithPolling(2*time.Second).Should(Succeed(), "Finalizers are not rebuilt as expected")
+}
+
+// removeFinalizers removes all Finalizers from objects in the owner graph.
+func removeFinalizers(ctx context.Context, proxy framework.ClusterProxy, namespace string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) {
+	graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath(), ownerGraphFilterFunction)
+	Expect(err).ToNot(HaveOccurred())
+	for _, object := range graph {
+		ref := object.Object
+		obj := new(unstructured.Unstructured)
+		obj.SetAPIVersion(ref.APIVersion)
+		obj.SetKind(ref.Kind)
+		obj.SetName(ref.Name)
+
+		Expect(proxy.GetClient().Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: object.Object.Name}, obj)).To(Succeed())
+		helper, err := patch.NewHelper(obj, proxy.GetClient())
+		Expect(err).ToNot(HaveOccurred())
+		obj.SetFinalizers([]string{})
+		Expect(helper.Patch(ctx, obj)).To(Succeed())
+	}
+}
+
+func getObjectsWithFinalizers(ctx context.Context, proxy framework.ClusterProxy, namespace string, allFinalizerAssertions map[string]func(name types.NamespacedName) []string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) (map[string]*unstructured.Unstructured, error) {
+	graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath(), ownerGraphFilterFunction)
+	if err != nil {
+		return nil, err
+	}
+
+	objsWithFinalizers := map[string]*unstructured.Unstructured{}
+
+	var allErrs []error
+	for _, node := range graph {
+		nodeNamespacedName := ctrlclient.ObjectKey{Namespace: node.Object.Namespace, Name: node.Object.Name}
+		obj := &unstructured.Unstructured{}
+		obj.SetAPIVersion(node.Object.APIVersion)
+		obj.SetKind(node.Object.Kind)
+		err = proxy.GetClient().Get(ctx, nodeNamespacedName, obj)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get object %s, %s", node.Object.Kind, klog.KRef(node.Object.Namespace, node.Object.Name))
+		}
+
+		// assert if the expected finalizers are set on the resource (including also checking if there are unexpected finalizers)
+		setFinalizers := obj.GetFinalizers()
+		var expectedFinalizers []string
+		if assertion, ok := allFinalizerAssertions[node.Object.Kind]; ok {
+			expectedFinalizers = assertion(types.NamespacedName{Namespace: node.Object.Namespace, Name: node.Object.Name})
+		}
+
+		if !sets.NewString(setFinalizers...).Equal(sets.NewString(expectedFinalizers...)) {
+			allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s, %s: expected: %v, found: %v",
+				node.Object.Kind, klog.KRef(node.Object.Namespace, node.Object.Name), expectedFinalizers, setFinalizers))
+		}
+		if len(setFinalizers) > 0 {
+			objsWithFinalizers[fmt.Sprintf("%s/%s/%s", node.Object.Kind, node.Object.Namespace, node.Object.Name)] = obj
+		}
+	}
+	return objsWithFinalizers, kerrors.NewAggregate(allErrs)
+}
+
+// assertFinalizersExist ensures that current Finalizers match those in the initialObjectsWithFinalizers.
+func assertFinalizersExist(ctx context.Context, proxy framework.ClusterProxy, namespace string, initialObjsWithFinalizers map[string]*unstructured.Unstructured, allFinalizerAssertions map[string]func(name types.NamespacedName) []string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) {
+	Eventually(func() error {
+		var allErrs []error
+		finalObjsWithFinalizers, _ := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction)
+
+		// Check if all the initial objects with finalizers have them back.
+		for objKindNamespacedName, obj := range initialObjsWithFinalizers {
+			// verify if finalizers for this resource were set on reconcile
+			if _, valid := finalObjsWithFinalizers[objKindNamespacedName]; !valid {
+				allErrs = append(allErrs, fmt.Errorf("no finalizers set for %s, at the beginning of the test it has %s",
+					objKindNamespacedName, obj.GetFinalizers()))
+				continue
+			}
+
+			// verify if this resource has the appropriate Finalizers set
+			expectedFinalizersF, assert := allFinalizerAssertions[obj.GetKind()]
+			// NOTE: this case should never happen because all the initialObjsWithFinalizers have been already checked
+			// against a finalizer assertion.
+			Expect(assert).To(BeTrue(), "finalizer assertions for %s are missing", objKindNamespacedName)
+			parts := strings.Split(objKindNamespacedName, "/")
+			expectedFinalizers := expectedFinalizersF(types.NamespacedName{Namespace: parts[1], Name: parts[2]})
+
+			setFinalizers := finalObjsWithFinalizers[objKindNamespacedName].GetFinalizers()
+			if !sets.NewString(setFinalizers...).Equal(sets.NewString(expectedFinalizers...)) {
+				allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s: expected: %v, found: %v",
+					objKindNamespacedName, expectedFinalizers, setFinalizers))
+			}
+		}
+
+		// Check if there are objects with finalizers not existing initially
+		for objKindNamespacedName, obj := range finalObjsWithFinalizers {
+			// verify if finalizers for this resource were set on reconcile
+			if _, valid := initialObjsWithFinalizers[objKindNamespacedName]; !valid {
+				allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s: expected: [], found: %v",
+					objKindNamespacedName, obj.GetFinalizers()))
+			}
+		}
+
+		return kerrors.NewAggregate(allErrs)
+	}).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).Should(Succeed())
+}
+
+// concatenateFinalizerAssertions concatenates all finalizer assertions into one map. It reports errors if assertions already exist.
+func concatenateFinalizerAssertions(finalizerAssertions ...map[string]func(name types.NamespacedName) []string) (map[string]func(name types.NamespacedName) []string, error) {
+	var allErrs []error
+	allFinalizerAssertions := make(map[string]func(name types.NamespacedName) []string, 0)
+
+	for i := range finalizerAssertions {
+		for kind, finalizers := range finalizerAssertions[i] {
+			if _, alreadyExists := allFinalizerAssertions[kind]; alreadyExists {
+				allErrs = append(allErrs, fmt.Errorf("finalizer assertion cannot be applied as it already exists for kind: %s", kind))
+				continue
+			}
+
+			allFinalizerAssertions[kind] = finalizers
+		}
+	}
+
+	return allFinalizerAssertions, kerrors.NewAggregate(allErrs)
+}
+
+// ------ dependency of the fork
+
+func setClusterPause(ctx context.Context, cli ctrlclient.Client, clusterKey types.NamespacedName, value bool) {
+	cluster := &clusterv1.Cluster{}
+	Expect(cli.Get(ctx, clusterKey, cluster)).To(Succeed())
+
+	pausePatch := ctrlclient.RawPatch(types.MergePatchType, []byte(fmt.Sprintf("{\"spec\":{\"paused\":%v}}", value)))
+	Expect(cli.Patch(ctx, cluster, pausePatch)).To(Succeed())
+}
+
+func byf(format string, a ...interface{}) {
+	By(fmt.Sprintf(format, a...))
+}