Merge pull request #4451 from zhzhuang-zju/unjoin

Enhanced forced deletion on karmadactl unjoin

pkg/karmadactl/unjoin/unjoin.go

@@ -147,7 +147,7 @@ func (j *CommandUnjoinOption) AddFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&j.ClusterKubeConfig, "cluster-kubeconfig", "",
 		"Path of the cluster's kubeconfig.")
 	flags.BoolVar(&j.forceDeletion, "force", false,
-		"Delete cluster and secret resources even if resources in the cluster targeted for unjoin are not removed successfully.")
+		"When set, the unjoin command will attempt to clean up resources in the member cluster before deleting the Cluster object. If the cleanup fails within the timeout period, the Cluster object will still be deleted, potentially leaving some resources behind in the member cluster.")
 	flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.")
 	flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.")
 }
@@ -181,9 +181,10 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error {
 // RunUnJoinCluster unJoin the cluster from karmada.
 func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error {
 	controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
+	controlPlaneKubeClient := kubeclient.NewForConfigOrDie(controlPlaneRestConfig)
 
 	// delete the cluster object in host cluster that associates the unjoining cluster
-	err := cmdutil.DeleteClusterObject(controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun)
+	err := cmdutil.DeleteClusterObject(controlPlaneKubeClient, controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion)
 	if err != nil {
 		klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
 		return err

pkg/karmadactl/unregister/unregister.go

@@ -130,6 +130,9 @@ type CommandUnregisterOption struct {
 	// ControlPlaneClient control plane client set
 	ControlPlaneClient karmadaclientset.Interface
 
+	// ControlPlaneKubeClient control plane kube client set
+	ControlPlaneKubeClient kubeclient.Interface
+
 	// MemberClusterClient member cluster client set
 	MemberClusterClient kubeclient.Interface
 }
@@ -223,6 +226,10 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromFile() error {
 	if err != nil {
 		return fmt.Errorf("failed to build karmada control plane clientset: %w", err)
 	}
+	j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg)
+	if err != nil {
+		return fmt.Errorf("failed to build kube control plane clientset: %w", err)
+	}
 	return nil
 }
 
@@ -248,7 +255,14 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromAgent() error {
 	}
 
 	j.ControlPlaneClient, err = register.ToKarmadaClient(karmadaCfg)
-	return err
+	if err != nil {
+		return fmt.Errorf("failed to build karmada control plane clientset: %w", err)
+	}
+	j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg)
+	if err != nil {
+		return fmt.Errorf("failed to build kube control plane clientset: %w", err)
+	}
+	return nil
 }
 
 func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment) (*clientcmdapi.Config, error) {
@@ -305,7 +319,8 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error {
 	}
 
 	// 1. delete the cluster object from the Karmada control plane
-	if err := cmdutil.DeleteClusterObject(j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun); err != nil {
+	//TODO: add flag --force to implement force deletion.
+	if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil {
 		klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
 		return err
 	}

pkg/karmadactl/util/cluster.go

@@ -21,17 +21,22 @@ import (
 	"fmt"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	kubeclient "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
+	"github.com/karmada-io/karmada/pkg/util"
+	"github.com/karmada-io/karmada/pkg/util/names"
 )
 
 // DeleteClusterObject deletes the cluster object from the Karmada control plane.
-func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string,
-	timeout time.Duration, dryRun bool) error {
+func DeleteClusterObject(controlPlaneKubeClient kubeclient.Interface, controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string,
+	timeout time.Duration, dryRun bool, forceDeletion bool) error {
 	if dryRun {
 		return nil
 	}
@@ -45,7 +50,8 @@ func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, c
 		return err
 	}
 
-	// make sure the given cluster object has been deleted
+	// make sure the given cluster object has been deleted.
+	// If the operation times out and `forceDeletion` is true, then force deletion begins, which involves sequentially deleting the `work`, `executionSpace`, and `cluster` finalizers.
 	err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) {
 		_, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{})
 		if apierrors.IsNotFound(err) {
@@ -58,10 +64,88 @@ func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, c
 		klog.Infof("Waiting for the cluster object %s to be deleted", clusterName)
 		return false, nil
 	})
+
+	// If the Cluster object not be deleted within the timeout period, it is likely due to the resources in the member
+	// cluster can not be cleaned up. With the option force deletion, we will try to clean up the Cluster object by
+	// removing the finalizers from related resources. This behavior may result in some resources remain in the member
+	// clusters.
+	if err != nil && forceDeletion {
+		klog.Warningf("Deleting the cluster object timed out. cluster name: %s, error: %v", clusterName, err)
+		klog.Infof("Start forced deletion. cluster name: %s", clusterName)
+		executionSpaceName := names.GenerateExecutionSpaceName(clusterName)
+		err = removeWorkFinalizer(executionSpaceName, controlPlaneKarmadaClient)
+		if err != nil {
+			klog.Errorf("Force deletion. Failed to remove the finalizer of Work, error: %v", err)
+		}
+
+		err = removeExecutionSpaceFinalizer(executionSpaceName, controlPlaneKubeClient)
+		if err != nil {
+			klog.Errorf("Force deletion. Failed to remove the finalizer of Namespace(%s), error: %v", executionSpaceName, err)
+		}
+
+		err = removeClusterFinalizer(clusterName, controlPlaneKarmadaClient)
+		if err != nil {
+			klog.Errorf("Force deletion. Failed to remove the finalizer of Cluster(%s), error: %v", clusterName, err)
+		}
+
+		klog.Infof("Forced deletion is complete.")
+		return nil
+	}
+
+	return err
+}
+
+// removeWorkFinalizer removes the finalizer of works in the executionSpace.
+func removeWorkFinalizer(executionSpaceName string, controlPlaneKarmadaClient karmadaclientset.Interface) error {
+	list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
-		klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err)
-		return err
+		return fmt.Errorf("failed to list work in executionSpace %s", executionSpaceName)
 	}
 
+	for i := range list.Items {
+		work := &list.Items[i]
+		if !controllerutil.ContainsFinalizer(work, util.ExecutionControllerFinalizer) {
+			continue
+		}
+		controllerutil.RemoveFinalizer(work, util.ExecutionControllerFinalizer)
+		_, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).Update(context.TODO(), work, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to remove the finalizer of work(%s/%s)", executionSpaceName, work.GetName())
+		}
+	}
 	return nil
 }
+
+// removeExecutionSpaceFinalizer removes the finalizer of executionSpace.
+func removeExecutionSpaceFinalizer(executionSpaceName string, controlPlaneKubeClient kubeclient.Interface) error {
+	executionSpace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), executionSpaceName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get Namespace(%s)", executionSpaceName)
+	}
+
+	if !controllerutil.ContainsFinalizer(executionSpace, string(corev1.FinalizerKubernetes)) {
+		return nil
+	}
+
+	controllerutil.RemoveFinalizer(executionSpace, "kubernetes")
+	_, err = controlPlaneKubeClient.CoreV1().Namespaces().Update(context.TODO(), executionSpace, metav1.UpdateOptions{})
+
+	return err
+}
+
+// removeClusterFinalizer removes the finalizer of cluster object.
+func removeClusterFinalizer(clusterName string, controlPlaneKarmadaClient karmadaclientset.Interface) error {
+	cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get Cluster(%s)", clusterName)
+	}
+
+	if !controllerutil.ContainsFinalizer(cluster, util.ClusterControllerFinalizer) {
+		return nil
+	}
+
+	controllerutil.RemoveFinalizer(cluster, util.ClusterControllerFinalizer)
+	_, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Update(context.TODO(), cluster, metav1.UpdateOptions{})
+
+	return err
+}