feat: remove finalizer when leave

cmd/mcs-controller-manager/main.go

@@ -9,6 +9,7 @@
 import (
 	"context"
 	"flag"
+	"go.goms.io/fleet-networking/pkg/common/controller"
 	"os"
 	"os/signal"
 	"sync"
@@ -39,8 +40,8 @@
 	fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"
 	"go.goms.io/fleet-networking/pkg/common/hubconfig"
 	imcv1alpha1 "go.goms.io/fleet-networking/pkg/controllers/member/internalmembercluster/v1alpha1"
 	imcv1beta1 "go.goms.io/fleet-networking/pkg/controllers/member/internalmembercluster/v1beta1"
 	"go.goms.io/fleet-networking/pkg/controllers/multiclusterservice"
 )
 
 var (
@@ -241,12 +242,13 @@
 	hubClient := hubMgr.GetClient()
 
 	klog.V(1).InfoS("Create multiclusterservice reconciler")
-	if err := (&multiclusterservice.Reconciler{
+	mcsController := multiclusterservice.Reconciler{
 		Client:               memberClient,
 		Scheme:               memberMgr.GetScheme(),
 		FleetSystemNamespace: *fleetSystemNamespace,
 		Recorder:             memberMgr.GetEventRecorderFor(multiclusterservice.ControllerName),
-	}).SetupWithManager(memberMgr); err != nil {
+	}
+	if err := mcsController.SetupWithManager(memberMgr); err != nil {
 		klog.ErrorS(err, "Unable to create multiclusterservice reconciler")
 		return err
 	}
@@ -269,6 +271,7 @@
 			MemberClient: memberClient,
 			HubClient:    hubClient,
 			AgentType:    clusterv1beta1.MultiClusterServiceAgent,
+			Controllers:  []controller.Controller{mcsController},
 		}).SetupWithManager(hubMgr); err != nil {
 			klog.ErrorS(err, "Unable to create internalmembercluster (v1beta1 API) reconciler")
 			return err

go.mod

@@ -18,6 +18,7 @@ require (
 require (
 	github.com/stretchr/testify v1.9.0
 	go.goms.io/fleet v0.10.5
+	golang.org/x/sync v0.7.0
 )
 
 require (

go.sum

@@ -120,6 +120,8 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/common/controller/controller.go

@@ -0,0 +1,9 @@
+package controller
+
+import "context"
+
+type MemberController interface {
+	Join(ctx context.Context) error
+
+	Leave(ctx context.Context) error
+}

pkg/controllers/hub/internalmembercluster/controller.go

@@ -0,0 +1,51 @@
+package internalmembercluster
+
+import (
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1"
+	"go.goms.io/fleet/pkg/utils/condition"
+)
+
+// Reconciler reconciles the distribution of EndpointSlices across the fleet.
+type Reconciler struct {
+	HubClient client.Client
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	imcKRef := klog.KRef(req.Namespace, req.Name)
+	startTime := time.Now()
+	klog.V(2).InfoS("Reconciliation starts", "internalMemberCluster", imcKRef)
+	defer func() {
+		latency := time.Since(startTime).Milliseconds()
+		klog.V(2).InfoS("Reconciliation ends", "internalMemberCluster", imcKRef, "latency", latency)
+	}()
+
+	var imc clusterv1beta1.InternalMemberCluster
+	if err := r.HubClient.Get(ctx, req.NamespacedName, &imc); err != nil {
+		if apierrors.IsNotFound(err) {
+			klog.V(4).InfoS("Internal member cluster object is not found", "internalMemberCluster", imcKRef)
+			return ctrl.Result{}, nil
+		}
+		klog.ErrorS(err, "Failed to get internal member cluster object", "internalMemberCluster", imcKRef)
+		return ctrl.Result{}, err
+	}
+
+	if imc.Spec.state != clusterv1beta1.ClusterStateLeave {
+		klog.V(2).Info("Skipping handling the internalMemberCluster non leave state", "internalMemberCluster", imcKRef, "clusterState", imc.Spec.State)
+		return ctrl.Result{}, nil
+	}
+
+	if condition.IsConditionStatusTrue(imc.GetConditionWithType(clusterv1beta1.MultiClusterServiceAgent, clusterv1beta1.AgentJoined))
+	 && condition.IsConditionStatusTrue(imc.GetConditionWithType(clusterv1beta1.ServiceExportImportAgent, clusterv1beta1.AgentJoined)) {
+		 // remove finalizer for internal member cluster if any in the reserved member namespace
+	}
+	return ctrl.Result{}, nil
+}
+
+func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&fleetnetv1alpha1.InternalMemberCluster{}).
+		Complete(r)
+}

pkg/controllers/member/endpointslice/controller.go

@@ -53,6 +53,7 @@ type Reconciler struct {
 	HubClient       client.Client
 	// The namespace reserved for the current member cluster in the hub cluster.
 	HubNamespace string
+	joined       *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=endpointsliceexports,verbs=get;list;watch;create;update;patch;delete

pkg/controllers/member/endpointsliceexport/controller.go

@@ -29,6 +29,7 @@ const (
 type Reconciler struct {
 	MemberClient client.Client
 	HubClient    client.Client
+	joined       *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=endpointsliceexports,verbs=get;list;watch;delete

pkg/controllers/member/endpointsliceimport/controller.go

@@ -82,6 +82,7 @@ type Reconciler struct {
 	HubClient       client.Client
 	// The namespace reserved for fleet resources in the member cluster.
 	FleetSystemNamespace string
+	joined               *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=endpointsliceimports,verbs=get;list;watch;update;patch

pkg/controllers/member/internalmembercluster/v1beta1/controller_v1beta1.go

@@ -8,8 +8,10 @@
 import (
 	"context"
 	"fmt"
+	"go.goms.io/fleet-networking/pkg/common/controller"
 	"time"
 
+	"golang.org/x/sync/errgroup"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,6 +40,7 @@
 	MemberClient client.Client
 	HubClient    client.Client
 	AgentType    clusterv1beta1.AgentType
+	Controllers  []controller.MemberController
 }
 
 //+kubebuilder:rbac:groups=fleet.azure.com,resources=internalmemberclusters,verbs=get;list;watch
@@ -84,9 +87,19 @@
 			}
 		}
 
+		if err := r.stopControllers(ctx); err != nil {
+			klog.ErrorS(err, "Failed to stop member controllers", "internalMemberCluster", imcKRef)
+			return ctrl.Result{}, err
+		}
+
 		// Update the agent status.
 		return ctrl.Result{}, r.updateAgentStatus(ctx, &imc)
 	case clusterv1beta1.ClusterStateJoin:
+		if err := r.startControllers(ctx); err != nil {
+			klog.ErrorS(err, "Failed to start member controllers", "internalMemberCluster", imcKRef)
+			return ctrl.Result{}, err
+		}
+
 		// The member cluster still has an active membership in the fleet; update the agent status.
 		if err := r.updateAgentStatus(ctx, &imc); err != nil {
 			return ctrl.Result{}, err
@@ -104,6 +117,26 @@
 	return ctrl.Result{}, nil
 }
 
+func (r *Reconciler) startControllers(ctx context.Context) {
+	errs, cctx := errgroup.WithContext(ctx)
+	for _, c := range r.Controllers {
+		errs.Go(func() error {
+			return c.Join(ctx)
+		})
+	}
+	return errs.Wait()
+}
+
+func (r *Reconciler) stopControllers(ctx context.Context) {
+	errs, cctx := errgroup.WithContext(ctx)
+	for _, c := range r.Controllers {
+		errs.Go(func() error {
+			return c.Leave(ctx)
+		})
+	}
+	return errs.Wait()
+}
+
 // updateAgentStatus reports the status of the agent via internal member cluster object.
 func (r *Reconciler) updateAgentStatus(ctx context.Context, imc *clusterv1beta1.InternalMemberCluster) error {
 	imcKObj := klog.KObj(imc)

pkg/controllers/member/internalserviceexport/controller.go

@@ -71,6 +71,7 @@ type Reconciler struct {
 	MemberClient    client.Client
 	HubClient       client.Client
 	Recorder        record.EventRecorder
+	joined          *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=internalserviceexports,verbs=get;list;watch;create;update;patch;delete

pkg/controllers/member/serviceexport/controller.go

@@ -52,6 +52,7 @@ type Reconciler struct {
 	// The namespace reserved for the current member cluster in the hub cluster.
 	HubNamespace string
 	Recorder     record.EventRecorder
+	joined       *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=serviceexports,verbs=get;list;watch;create;update;patch;delete

pkg/controllers/member/serviceimport/controller.go

@@ -36,6 +36,7 @@ type Reconciler struct {
 
 	HubClient    client.Client
 	MemberClient client.Client
+	joined       *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=serviceimports,verbs=get;list;watch;update;patch

pkg/controllers/multiclusterservice/controller.go

@@ -63,6 +63,7 @@
 	Scheme               *runtime.Scheme
 	FleetSystemNamespace string // reserved fleet namespace
 	Recorder             record.EventRecorder
+	joined               *atomic.Bool
 }
 
 //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
@@ -447,3 +448,20 @@
 		}
 	}
 }
+
+// Join marks the joined status as true.
+func (r *Reconciler) Join(_ context.Context) error {
+	if !r.joined.Load() {
+		klog.InfoS("Mark the multiClusterService controller joined")
+	}
+	r.joined.Store(true)
+}
+
+// Leave marks the joined status as false.
+// When the controller is in the leave state, it will only handle the delete events.
+func (r *ApplyWorkReconciler) Leave(_ context.Context) error {
+	if r.joined.Load() {
+		klog.InfoS("Mark the multiClusterService controller left")
+	}
+	r.joined.Store(false)
+}