From b27fb8041cf3c352fc6e44bc52ae1ffeb54cbacc Mon Sep 17 00:00:00 2001
From: wulemao <1194736083@qq.com>
Date: Mon, 23 Sep 2024 20:42:56 +0800
Subject: [PATCH] support preserving resource in member cluster when executing
 `karmadactl unjoin`

---
 .../execution/execution_controller.go         |  7 ++-
 pkg/karmadactl/unjoin/unjoin.go               | 56 +++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/pkg/controllers/execution/execution_controller.go b/pkg/controllers/execution/execution_controller.go
index 679e8560b890..41b50d67b216 100644
--- a/pkg/controllers/execution/execution_controller.go
+++ b/pkg/controllers/execution/execution_controller.go
@@ -103,7 +103,7 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 
 	if !work.DeletionTimestamp.IsZero() {
 		// Abort deleting workload if cluster is unready when unjoining cluster, otherwise the unjoin process will be failed.
-		if util.IsClusterReady(&cluster.Status) {
+		if util.IsClusterReady(&cluster.Status) && !isWorkloadPreservedOnDeletion(work) {
 			err := c.tryDeleteWorkload(ctx, clusterName, work)
 			if err != nil {
 				klog.Errorf("Failed to delete work %v, namespace is %v, err is %v", work.Name, work.Namespace, err)
@@ -329,3 +329,8 @@ func (c *Controller) eventf(object *unstructured.Unstructured, eventType, reason
 	}
 	c.EventRecorder.Eventf(ref, eventType, reason, messageFmt, args...)
 }
+
+// isWorkloadPreservedOnDeletion judge whether resources should be preserved on the member cluster.
+func isWorkloadPreservedOnDeletion(work *workv1alpha1.Work) bool {
+	return work.Spec.PreserveResourcesOnDeletion != nil && *work.Spec.PreserveResourcesOnDeletion
+}
diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go
index 4e0fa4450b0e..0cc57da38fc0 100644
--- a/pkg/karmadactl/unjoin/unjoin.go
+++ b/pkg/karmadactl/unjoin/unjoin.go
@@ -18,6 +18,7 @@ package unjoin
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -25,6 +26,7 @@ import (
 	"github.com/spf13/pflag"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kubeclient "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -51,6 +53,9 @@ var (
 		# Unjoin cluster from karmada control plane and attempt to remove resources created by karmada in the unjoining cluster
 		%[1]s unjoin CLUSTER_NAME --cluster-kubeconfig=<KUBECONFIG>
 
+        # Unjoin cluster from karmada control plane and keep users' resources preserved in the unjoining cluster
+		%[1]s unjoin CLUSTER_NAME --cluster-kubeconfig=<KUBECONFIG> --preserve-resources=true
+
 		# Unjoin cluster from karmada control plane with timeout
 		%[1]s unjoin CLUSTER_NAME --cluster-kubeconfig=<KUBECONFIG> --wait 2m`)
 )
@@ -108,6 +113,9 @@ type CommandUnjoinOption struct {
 	// DryRun tells if run the command in dry-run mode, without making any server requests.
 	DryRun bool
 
+	// PreserveResources tells if users' resources should be preserved in the unjoining cluster.
+	PreserveResources bool
+
 	forceDeletion bool
 
 	// Wait tells maximum command execution time
@@ -152,6 +160,7 @@ func (j *CommandUnjoinOption) AddFlags(flags *pflag.FlagSet) {
 		"Path of the cluster's kubeconfig.")
 	flags.BoolVar(&j.forceDeletion, "force", false,
 		"Delete cluster and secret resources even if resources in the cluster targeted for unjoin are not removed successfully.")
+	flags.BoolVar(&j.PreserveResources, "preserve-resources", false, "control whether users' resources should be preserved in the unjoining cluster.")
 	flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.")
 	flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.")
 }
@@ -186,6 +195,16 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error {
 func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error {
 	controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
 
+	// Attempt to mark the cluster related work as `preserveResourcesOnDeletion`
+	// if user set command argument `preserve-resources` to true
+	if j.PreserveResources {
+		err := j.markClusterRelatedWorkAsPreserved(controlPlaneKarmadaClient, j.ClusterName)
+		if err != nil {
+			klog.Errorf("Failed to mark the work in cluster %s as preserve-resources", j.ClusterName)
+			return err
+		}
+	}
+
 	// delete the cluster object in host cluster that associates the unjoining cluster
 	err := j.deleteClusterObject(controlPlaneKarmadaClient)
 	if err != nil {
@@ -225,6 +244,43 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo
 	return nil
 }
 
+// markClusterRelatedWorkAsPreserved attempt to mark the cluster related work as `preserveResourcesOnDeletion`
+func (j *CommandUnjoinOption) markClusterRelatedWorkAsPreserved(controlPlaneKarmadaClient *karmadaclientset.Clientset, clusterName string) error {
+	if j.DryRun {
+		return nil
+	}
+
+	patch := []map[string]interface{}{
+		{
+			"op":    "replace",
+			"path":  "/spec/preserveResourcesOnDeletion",
+			"value": true,
+		},
+	}
+	patchBytes, err := json.Marshal(patch)
+	if err != nil {
+		klog.Errorf("Failed to marshal patch bytes, error: %+v", err)
+		return err
+	}
+
+	workNamespace := names.GenerateExecutionSpaceName(clusterName)
+	workList, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(workNamespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		klog.Errorf("Failed to list works in cluster %s, error: %+v", clusterName, err)
+		return err
+	}
+
+	for _, work := range workList.Items {
+		_, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(workNamespace).Patch(context.TODO(), work.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		if err != nil {
+			klog.Errorf("Failed to set preserveResourcesOnDeletion in work(%s/%s), error: %+v", workNamespace, work.Name, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
 // deleteClusterObject delete the cluster object in host cluster that associates the unjoining cluster
 func (j *CommandUnjoinOption) deleteClusterObject(controlPlaneKarmadaClient *karmadaclientset.Clientset) error {
 	if j.DryRun {