diff --git a/test/e2e/ownerrefs_finalizers_test.go b/test/e2e/ownerrefs_finalizers_test.go index 847da6f26b..caba381351 100644 --- a/test/e2e/ownerrefs_finalizers_test.go +++ b/test/e2e/ownerrefs_finalizers_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -28,8 +29,10 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/utils/ptr" @@ -101,7 +104,7 @@ var _ = Describe("Ensure OwnerReferences and Finalizers are resilient with Failu ) // This check ensures that finalizers are resilient - i.e. correctly re-reconciled, when removed. By("Checking that finalizers are resilient") - framework.ValidateFinalizersResilience(ctx, proxy, namespace, clusterName, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName), + ValidateFinalizersResilience(ctx, proxy, namespace, clusterName, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName), framework.CoreFinalizersAssertionWithLegacyClusters, framework.KubeadmControlPlaneFinalizersAssertion, framework.ExpFinalizersAssertion, @@ -339,3 +342,171 @@ func forcePeriodicReconcile(ctx context.Context, c ctrlclient.Client, namespace } }() } + +// ------ TMP fork to test things in CAPI quickly + +// ValidateFinalizersResilience checks that expected finalizers are in place, deletes them, and verifies that expected finalizers are properly added again. +func ValidateFinalizersResilience(ctx context.Context, proxy framework.ClusterProxy, namespace, clusterName string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction, finalizerAssertions ...map[string]func(name types.NamespacedName) []string) { + clusterKey := ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName} + allFinalizerAssertions, err := concatenateFinalizerAssertions(finalizerAssertions...) + Expect(err).ToNot(HaveOccurred()) + + // Collect all objects where finalizers were initially set + byf("Check that the finalizers are as expected") + _, err = getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction) + Expect(err).ToNot(HaveOccurred(), "Finalizers are not as expected") + + byf("Removing all the finalizers") + // Setting the paused property on the Cluster resource will pause reconciliations, thereby having no effect on Finalizers. + // This also makes debugging easier. + setClusterPause(ctx, proxy.GetClient(), clusterKey, true) + + // We are testing the worst-case scenario, i.e. all finalizers are deleted. + // Once all Clusters are paused remove all the Finalizers from all objects in the graph. + // The reconciliation loop should be able to recover from this, by adding the required Finalizers back. + removeFinalizers(ctx, proxy, namespace, ownerGraphFilterFunction) + + // Unpause the cluster. + setClusterPause(ctx, proxy.GetClient(), clusterKey, false) + + // Check that the Finalizers are as expected after further reconciliations. + byf("Check that the finalizers are rebuilt as expected") + // assertFinalizersExist(ctx, proxy, namespace, objectsWithFinalizers, allFinalizerAssertions, ownerGraphFilterFunction) + Eventually(func() error { + _, err := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction) + + return err + }).WithTimeout(1*time.Minute).WithPolling(2*time.Second).Should(Succeed(), "Finalizers are not rebuilt as expected") +} + +// removeFinalizers removes all Finalizers from objects in the owner graph. +func removeFinalizers(ctx context.Context, proxy framework.ClusterProxy, namespace string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) { + graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath(), ownerGraphFilterFunction) + Expect(err).ToNot(HaveOccurred()) + for _, object := range graph { + ref := object.Object + obj := new(unstructured.Unstructured) + obj.SetAPIVersion(ref.APIVersion) + obj.SetKind(ref.Kind) + obj.SetName(ref.Name) + + Expect(proxy.GetClient().Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: object.Object.Name}, obj)).To(Succeed()) + helper, err := patch.NewHelper(obj, proxy.GetClient()) + Expect(err).ToNot(HaveOccurred()) + obj.SetFinalizers([]string{}) + Expect(helper.Patch(ctx, obj)).To(Succeed()) + } +} + +func getObjectsWithFinalizers(ctx context.Context, proxy framework.ClusterProxy, namespace string, allFinalizerAssertions map[string]func(name types.NamespacedName) []string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) (map[string]*unstructured.Unstructured, error) { + graph, err := clusterctlcluster.GetOwnerGraph(ctx, namespace, proxy.GetKubeconfigPath(), ownerGraphFilterFunction) + if err != nil { + return nil, err + } + + objsWithFinalizers := map[string]*unstructured.Unstructured{} + + var allErrs []error + for _, node := range graph { + nodeNamespacedName := ctrlclient.ObjectKey{Namespace: node.Object.Namespace, Name: node.Object.Name} + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(node.Object.APIVersion) + obj.SetKind(node.Object.Kind) + err = proxy.GetClient().Get(ctx, nodeNamespacedName, obj) + if err != nil { + return nil, errors.Wrapf(err, "failed to get object %s, %s", node.Object.Kind, klog.KRef(node.Object.Namespace, node.Object.Name)) + } + + // assert if the expected finalizers are set on the resource (including also checking if there are unexpected finalizers) + setFinalizers := obj.GetFinalizers() + var expectedFinalizers []string + if assertion, ok := allFinalizerAssertions[node.Object.Kind]; ok { + expectedFinalizers = assertion(types.NamespacedName{Namespace: node.Object.Namespace, Name: node.Object.Name}) + } + + if !sets.NewString(setFinalizers...).Equal(sets.NewString(expectedFinalizers...)) { + allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s, %s: expected: %v, found: %v", + node.Object.Kind, klog.KRef(node.Object.Namespace, node.Object.Name), expectedFinalizers, setFinalizers)) + } + if len(setFinalizers) > 0 { + objsWithFinalizers[fmt.Sprintf("%s/%s/%s", node.Object.Kind, node.Object.Namespace, node.Object.Name)] = obj + } + } + return objsWithFinalizers, kerrors.NewAggregate(allErrs) +} + +// assertFinalizersExist ensures that current Finalizers match those in the initialObjectsWithFinalizers. +func assertFinalizersExist(ctx context.Context, proxy framework.ClusterProxy, namespace string, initialObjsWithFinalizers map[string]*unstructured.Unstructured, allFinalizerAssertions map[string]func(name types.NamespacedName) []string, ownerGraphFilterFunction clusterctlcluster.GetOwnerGraphFilterFunction) { + Eventually(func() error { + var allErrs []error + finalObjsWithFinalizers, _ := getObjectsWithFinalizers(ctx, proxy, namespace, allFinalizerAssertions, ownerGraphFilterFunction) + + // Check if all the initial objects with finalizers have them back. + for objKindNamespacedName, obj := range initialObjsWithFinalizers { + // verify if finalizers for this resource were set on reconcile + if _, valid := finalObjsWithFinalizers[objKindNamespacedName]; !valid { + allErrs = append(allErrs, fmt.Errorf("no finalizers set for %s, at the beginning of the test it has %s", + objKindNamespacedName, obj.GetFinalizers())) + continue + } + + // verify if this resource has the appropriate Finalizers set + expectedFinalizersF, assert := allFinalizerAssertions[obj.GetKind()] + // NOTE: this case should never happen because all the initialObjsWithFinalizers have been already checked + // against a finalizer assertion. + Expect(assert).To(BeTrue(), "finalizer assertions for %s are missing", objKindNamespacedName) + parts := strings.Split(objKindNamespacedName, "/") + expectedFinalizers := expectedFinalizersF(types.NamespacedName{Namespace: parts[1], Name: parts[2]}) + + setFinalizers := finalObjsWithFinalizers[objKindNamespacedName].GetFinalizers() + if !sets.NewString(setFinalizers...).Equal(sets.NewString(expectedFinalizers...)) { + allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s: expected: %v, found: %v", + objKindNamespacedName, expectedFinalizers, setFinalizers)) + } + } + + // Check if there are objects with finalizers not existing initially + for objKindNamespacedName, obj := range finalObjsWithFinalizers { + // verify if finalizers for this resource were set on reconcile + if _, valid := initialObjsWithFinalizers[objKindNamespacedName]; !valid { + allErrs = append(allErrs, fmt.Errorf("unexpected finalizers for %s: expected: [], found: %v", + objKindNamespacedName, obj.GetFinalizers())) + } + } + + return kerrors.NewAggregate(allErrs) + }).WithTimeout(1 * time.Minute).WithPolling(2 * time.Second).Should(Succeed()) +} + +// concatenateFinalizerAssertions concatenates all finalizer assertions into one map. It reports errors if assertions already exist. +func concatenateFinalizerAssertions(finalizerAssertions ...map[string]func(name types.NamespacedName) []string) (map[string]func(name types.NamespacedName) []string, error) { + var allErrs []error + allFinalizerAssertions := make(map[string]func(name types.NamespacedName) []string, 0) + + for i := range finalizerAssertions { + for kind, finalizers := range finalizerAssertions[i] { + if _, alreadyExists := allFinalizerAssertions[kind]; alreadyExists { + allErrs = append(allErrs, fmt.Errorf("finalizer assertion cannot be applied as it already exists for kind: %s", kind)) + continue + } + + allFinalizerAssertions[kind] = finalizers + } + } + + return allFinalizerAssertions, kerrors.NewAggregate(allErrs) +} + +// ------ dependency of the fork + +func setClusterPause(ctx context.Context, cli ctrlclient.Client, clusterKey types.NamespacedName, value bool) { + cluster := &clusterv1.Cluster{} + Expect(cli.Get(ctx, clusterKey, cluster)).To(Succeed()) + + pausePatch := ctrlclient.RawPatch(types.MergePatchType, []byte(fmt.Sprintf("{\"spec\":{\"paused\":%v}}", value))) + Expect(cli.Patch(ctx, cluster, pausePatch)).To(Succeed()) +} + +func byf(format string, a ...interface{}) { + By(fmt.Sprintf(format, a...)) +}