From 10e98accbf9ae37e057c29041adb0c49bcebf695 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Wed, 11 Dec 2024 16:42:30 +0800 Subject: [PATCH] karmadactl unregister: force unregister pull mode member clusters with the flag --force Signed-off-by: zhzhuang-zju --- pkg/karmadactl/unregister/unregister.go | 9 +- pkg/karmadactl/util/cluster.go | 64 +------- pkg/karmadactl/util/finalizer.go | 85 +++++++++++ pkg/karmadactl/util/finalizer_test.go | 192 ++++++++++++++++++++++++ pkg/karmadactl/util/work.go | 19 ++- 5 files changed, 304 insertions(+), 65 deletions(-) create mode 100644 pkg/karmadactl/util/finalizer.go create mode 100644 pkg/karmadactl/util/finalizer_test.go diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go index 353bb8df3b6d..5e5dac5d7b8c 100644 --- a/pkg/karmadactl/unregister/unregister.go +++ b/pkg/karmadactl/unregister/unregister.go @@ -138,6 +138,8 @@ type CommandUnregisterOption struct { // rbacResources contains RBAC resources that grant the necessary permissions for the unregistering cluster to access to Karmada control plane. rbacResources *register.RBACResources + + forceDeletion bool } // AddFlags adds flags to the specified FlagSet. @@ -152,6 +154,8 @@ func (j *CommandUnregisterOption) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&j.ClusterNamespace, "cluster-namespace", options.DefaultKarmadaClusterNamespace, "Namespace in the control plane where member cluster secrets are stored.") flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.") flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") + flags.BoolVar(&j.forceDeletion, "force", false, + "When set, the unregister command will attempt to clean up resources in the member cluster before deleting the Cluster object. If the cleanup fails within the timeout period, the Cluster object will still be deleted, potentially leaving some resources behind in the member cluster.") } // Complete ensures that options are valid and marshals them if necessary. @@ -318,7 +322,7 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { // 1. delete the work object from the Karmada control plane // When deleting a cluster, the deletion triggers the removal of executionSpace, which can lead to the deletion of RBAC roles related to work. // Therefore, the deletion of work should be performed before deleting the cluster. - err := cmdutil.EnsureWorksDeleted(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait) + err := cmdutil.EnsureWorksDeleted(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait, j.forceDeletion) if err != nil { klog.Errorf("Failed to delete works object. cluster name: %s, error: %v", j.ClusterName, err) return err @@ -326,8 +330,7 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { j.Wait = j.Wait - time.Since(start) // 2. delete the cluster object from the Karmada control plane - //TODO: add flag --force to implement force deletion. - if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { + if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } diff --git a/pkg/karmadactl/util/cluster.go b/pkg/karmadactl/util/cluster.go index f5da3f4fab35..d93232675acf 100644 --- a/pkg/karmadactl/util/cluster.go +++ b/pkg/karmadactl/util/cluster.go @@ -21,16 +21,13 @@ import ( "fmt" "time" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" - "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -73,17 +70,17 @@ func DeleteClusterObject(controlPlaneKubeClient kubeclient.Interface, controlPla klog.Warningf("Deleting the cluster object timed out. cluster name: %s, error: %v", clusterName, err) klog.Infof("Start forced deletion. cluster name: %s", clusterName) executionSpaceName := names.GenerateExecutionSpaceName(clusterName) - err = removeWorkFinalizer(executionSpaceName, controlPlaneKarmadaClient) + err = RemoveWorkFinalizer(executionSpaceName, controlPlaneKarmadaClient) if err != nil { klog.Errorf("Force deletion. Failed to remove the finalizer of Work, error: %v", err) } - err = removeExecutionSpaceFinalizer(executionSpaceName, controlPlaneKubeClient) + err = RemoveExecutionSpaceFinalizer(executionSpaceName, controlPlaneKubeClient) if err != nil { klog.Errorf("Force deletion. Failed to remove the finalizer of Namespace(%s), error: %v", executionSpaceName, err) } - err = removeClusterFinalizer(clusterName, controlPlaneKarmadaClient) + err = RemoveClusterFinalizer(clusterName, controlPlaneKarmadaClient) if err != nil { klog.Errorf("Force deletion. Failed to remove the finalizer of Cluster(%s), error: %v", clusterName, err) } @@ -94,58 +91,3 @@ func DeleteClusterObject(controlPlaneKubeClient kubeclient.Interface, controlPla return err } - -// removeWorkFinalizer removes the finalizer of works in the executionSpace. -func removeWorkFinalizer(executionSpaceName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { - list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return fmt.Errorf("failed to list work in executionSpace %s", executionSpaceName) - } - - for i := range list.Items { - work := &list.Items[i] - if !controllerutil.ContainsFinalizer(work, util.ExecutionControllerFinalizer) { - continue - } - controllerutil.RemoveFinalizer(work, util.ExecutionControllerFinalizer) - _, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).Update(context.TODO(), work, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to remove the finalizer of work(%s/%s)", executionSpaceName, work.GetName()) - } - } - return nil -} - -// removeExecutionSpaceFinalizer removes the finalizer of executionSpace. -func removeExecutionSpaceFinalizer(executionSpaceName string, controlPlaneKubeClient kubeclient.Interface) error { - executionSpace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), executionSpaceName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get Namespace(%s)", executionSpaceName) - } - - if !controllerutil.ContainsFinalizer(executionSpace, string(corev1.FinalizerKubernetes)) { - return nil - } - - controllerutil.RemoveFinalizer(executionSpace, "kubernetes") - _, err = controlPlaneKubeClient.CoreV1().Namespaces().Update(context.TODO(), executionSpace, metav1.UpdateOptions{}) - - return err -} - -// removeClusterFinalizer removes the finalizer of cluster object. -func removeClusterFinalizer(clusterName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { - cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("failed to get Cluster(%s)", clusterName) - } - - if !controllerutil.ContainsFinalizer(cluster, util.ClusterControllerFinalizer) { - return nil - } - - controllerutil.RemoveFinalizer(cluster, util.ClusterControllerFinalizer) - _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Update(context.TODO(), cluster, metav1.UpdateOptions{}) - - return err -} diff --git a/pkg/karmadactl/util/finalizer.go b/pkg/karmadactl/util/finalizer.go new file mode 100644 index 000000000000..4b106771c552 --- /dev/null +++ b/pkg/karmadactl/util/finalizer.go @@ -0,0 +1,85 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + "github.com/karmada-io/karmada/pkg/util" +) + +// RemoveWorkFinalizer removes the finalizer of works in the executionSpace. +func RemoveWorkFinalizer(executionSpaceName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list work in executionSpace %s", executionSpaceName) + } + + for i := range list.Items { + work := &list.Items[i] + if !controllerutil.ContainsFinalizer(work, util.ExecutionControllerFinalizer) { + continue + } + controllerutil.RemoveFinalizer(work, util.ExecutionControllerFinalizer) + _, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).Update(context.TODO(), work, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to remove the finalizer of work(%s/%s)", executionSpaceName, work.GetName()) + } + } + return nil +} + +// RemoveExecutionSpaceFinalizer removes the finalizer of executionSpace. +func RemoveExecutionSpaceFinalizer(executionSpaceName string, controlPlaneKubeClient kubeclient.Interface) error { + executionSpace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), executionSpaceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Namespace(%s)", executionSpaceName) + } + + if !controllerutil.ContainsFinalizer(executionSpace, string(corev1.FinalizerKubernetes)) { + return nil + } + + controllerutil.RemoveFinalizer(executionSpace, "kubernetes") + _, err = controlPlaneKubeClient.CoreV1().Namespaces().Update(context.TODO(), executionSpace, metav1.UpdateOptions{}) + + return err +} + +// RemoveClusterFinalizer removes the finalizer of cluster object. +func RemoveClusterFinalizer(clusterName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Cluster(%s)", clusterName) + } + + if !controllerutil.ContainsFinalizer(cluster, util.ClusterControllerFinalizer) { + return nil + } + + controllerutil.RemoveFinalizer(cluster, util.ClusterControllerFinalizer) + _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Update(context.TODO(), cluster, metav1.UpdateOptions{}) + + return err +} diff --git a/pkg/karmadactl/util/finalizer_test.go b/pkg/karmadactl/util/finalizer_test.go new file mode 100644 index 000000000000..895daf2a45a6 --- /dev/null +++ b/pkg/karmadactl/util/finalizer_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" + fakekarmadaclient "github.com/karmada-io/karmada/pkg/generated/clientset/versioned/fake" + "github.com/karmada-io/karmada/pkg/util" +) + +var testFinalizer = "test.io/test" + +func TestRemoveWorkFinalizer(t *testing.T) { + testItems := []struct { + name string + executionSpace string + workList []runtime.Object + wantErr bool + expect map[string][]string + }{ + { + name: "empty work list", + executionSpace: "test", + workList: []runtime.Object{}, + wantErr: false, + expect: nil, + }, + { + name: "work without finalizer", + executionSpace: "test", + workList: []runtime.Object{ + &workv1alpha1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work1", + Namespace: "test", + }, + }, + }, + wantErr: false, + expect: map[string][]string{"work1": {}}, + }, + { + name: "work with ExecutionControllerFinalizer", + executionSpace: "test", + workList: []runtime.Object{ + &workv1alpha1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work1", + Namespace: "test", + Finalizers: []string{util.ExecutionControllerFinalizer, testFinalizer}, + }, + }, + &workv1alpha1.Work{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work2", + Namespace: "test", + Finalizers: []string{util.ExecutionControllerFinalizer}, + }, + }, + }, + wantErr: false, + expect: map[string][]string{"work1": {testFinalizer}, "work2": {}}, + }, + } + for _, tt := range testItems { + t.Run(tt.name, func(t *testing.T) { + controlPlaneKarmadaClient := fakekarmadaclient.NewSimpleClientset(tt.workList...) + err := RemoveWorkFinalizer(tt.executionSpace, controlPlaneKarmadaClient) + assert.Equal(t, tt.wantErr, err != nil) + + for name, finalizers := range tt.expect { + work, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(tt.executionSpace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + assert.ElementsMatch(t, finalizers, work.GetFinalizers()) + } + }) + } +} + +func TestRemoveExecutionSpaceFinalizer(t *testing.T) { + testItems := []struct { + name string + executionSpace corev1.Namespace + wantErr bool + expectFinalizers []string + }{ + { + name: "executionSpace without finalizer", + executionSpace: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespace1", + }, + }, + wantErr: false, + expectFinalizers: []string{}, + }, + { + name: "executionSpace with FinalizerKubernetes", + executionSpace: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "namespace1", + Finalizers: []string{string(corev1.FinalizerKubernetes), testFinalizer}, + }, + }, + wantErr: false, + expectFinalizers: []string{testFinalizer}, + }, + } + for _, tt := range testItems { + t.Run(tt.name, func(t *testing.T) { + controlPlaneKubeClient := fake.NewClientset(&tt.executionSpace) + err := RemoveExecutionSpaceFinalizer(tt.executionSpace.Name, controlPlaneKubeClient) + assert.Equal(t, tt.wantErr, err != nil) + + namespace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), tt.executionSpace.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + assert.ElementsMatch(t, tt.expectFinalizers, namespace.GetFinalizers()) + }) + } +} + +func TestRemoveClusterFinalizer(t *testing.T) { + testItems := []struct { + name string + cluster clusterv1alpha1.Cluster + wantErr bool + expectFinalizers []string + }{ + { + name: "cluster without finalizer", + cluster: clusterv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + }, + }, + wantErr: false, + expectFinalizers: []string{}, + }, + { + name: "cluster with ClusterControllerFinalizer", + cluster: clusterv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Finalizers: []string{util.ClusterControllerFinalizer, testFinalizer}, + }, + }, + wantErr: false, + expectFinalizers: []string{testFinalizer}, + }, + } + for _, tt := range testItems { + t.Run(tt.name, func(t *testing.T) { + controlPlaneKarmadaClient := fakekarmadaclient.NewSimpleClientset(&tt.cluster) + err := RemoveClusterFinalizer(tt.cluster.Name, controlPlaneKarmadaClient) + assert.Equal(t, tt.wantErr, err != nil) + + cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), tt.cluster.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + assert.ElementsMatch(t, tt.expectFinalizers, cluster.GetFinalizers()) + }) + } +} diff --git a/pkg/karmadactl/util/work.go b/pkg/karmadactl/util/work.go index 48fe759bb99d..40debeb03839 100644 --- a/pkg/karmadactl/util/work.go +++ b/pkg/karmadactl/util/work.go @@ -24,13 +24,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" ) // EnsureWorksDeleted ensures that all Work resources in the specified namespace are deleted. func EnsureWorksDeleted(controlPlaneKarmadaClient karmadaclientset.Interface, namespace string, - timeout time.Duration) error { + timeout time.Duration, forceDeletion bool) error { // make sure the works object under the given namespace has been deleted. err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).List(context.TODO(), metav1.ListOptions{}) @@ -51,5 +52,21 @@ func EnsureWorksDeleted(controlPlaneKarmadaClient karmadaclientset.Interface, na return false, nil }) + // If the Works not be deleted within the timeout period, it is likely due to the resources in the member + // cluster can not be cleaned up. With the option force deletion, we will try to clean up the Works object by + // removing the finalizers from related resources. This behavior may result in some resources remain in the member + // clusters. + if err != nil && forceDeletion { + klog.Warningf("Deleting the work object timed out. ExecutionSpace: %s, error: %v", namespace, err) + klog.Infof("Start forced deletion. Deleting finalizer of works in ExecutionSpace: %s", namespace) + err = RemoveWorkFinalizer(namespace, controlPlaneKarmadaClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Work, error: %v", err) + } + + klog.Infof("Work object force deletion is complete.") + return nil + } + return err }