From 08ad3144a81a861060b85c05458c5b53c1e2562b Mon Sep 17 00:00:00 2001 From: Erik Merkle Date: Wed, 3 May 2023 12:46:40 -0500 Subject: [PATCH 1/4] Refactor findDcForDeletion, findReaperForDeletion and findStargateForDeletion --- controllers/k8ssandra/cleanup.go | 123 +++++++++++++++++++------------ 1 file changed, 76 insertions(+), 47 deletions(-) diff --git a/controllers/k8ssandra/cleanup.go b/controllers/k8ssandra/cleanup.go index 4baedefe2..34b8a5603 100644 --- a/controllers/k8ssandra/cleanup.go +++ b/controllers/k8ssandra/cleanup.go @@ -218,34 +218,22 @@ func (r *K8ssandraClusterReconciler) findStargateForDeletion( dcName string, remoteClient client.Client) (*stargateapi.Stargate, client.Client, error) { - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - stargateList := &stargateapi.StargateList{} stargateName := kcKey.Name + "-" + dcName + "-stargate" - if remoteClient == nil { for _, remoteClient := range r.ClientCache.GetAllClients() { - err := remoteClient.List(ctx, stargateList, options) + stargate, err := finStargateForDeletionWithRemoteClient(ctx, kcKey, dcName, stargateName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) } - for _, stargate := range stargateList.Items { - if stargate.Name == stargateName { - return &stargate, remoteClient, nil - } - } + return stargate, remoteClient, nil } } else { - err := remoteClient.List(ctx, stargateList, options) + stargate, err := finStargateForDeletionWithRemoteClient(ctx, kcKey, dcName, stargateName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) } - for _, stargate := range stargateList.Items { - if stargate.Name == stargateName { - return &stargate, remoteClient, nil - } - } + return stargate, remoteClient, nil } return nil, nil, nil @@ -257,34 +245,22 @@ func (r *K8ssandraClusterReconciler) findReaperForDeletion( dcName string, remoteClient client.Client) (*reaperapi.Reaper, client.Client, error) { - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - reaperList := &reaperapi.ReaperList{} reaperName := kcKey.Name + "-" + dcName + "-reaper" - if remoteClient == nil { for _, remoteClient := range r.ClientCache.GetAllClients() { - err := remoteClient.List(ctx, reaperList, options) + reaper, err := finReaperForDeletionWithRemoteClient(ctx, kcKey, dcName, reaperName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) } - for _, reaper := range reaperList.Items { - if reaper.Name == reaperName { - return &reaper, remoteClient, nil - } - } + return reaper, remoteClient, nil } } else { - err := remoteClient.List(ctx, reaperList, options) + reaper, err := finReaperForDeletionWithRemoteClient(ctx, kcKey, dcName, reaperName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) } - for _, reaper := range reaperList.Items { - if reaper.Name == reaperName { - return &reaper, remoteClient, nil - } - } + return reaper, remoteClient, nil } return nil, nil, nil @@ -295,33 +271,22 @@ func (r *K8ssandraClusterReconciler) findDcForDeletion( kcKey client.ObjectKey, dcName string, remoteClient client.Client) (*cassdcapi.CassandraDatacenter, client.Client, error) { - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - dcList := &cassdcapi.CassandraDatacenterList{} if remoteClient == nil { for _, remoteClient := range r.ClientCache.GetAllClients() { - err := remoteClient.List(ctx, dcList, options) + dc, err := findDcForDeletionWithRemoteClient(ctx, kcKey, dcName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to CassandraDatacenter (%s) for DC (%s) deletion: %v", dcName, dcName, err) } - for _, dc := range dcList.Items { - if dc.Name == dcName { - return &dc, remoteClient, nil - } - } + return dc, remoteClient, nil } } else { - err := remoteClient.List(ctx, dcList, options) + dc, err := findDcForDeletionWithRemoteClient(ctx, kcKey, dcName, remoteClient) if err != nil { return nil, nil, fmt.Errorf("failed to find CassandraDatacenter (%s) for deletion: %v", dcName, err) } - for _, dc := range dcList.Items { - if dc.Name == dcName { - return &dc, remoteClient, nil - } - } + return dc, remoteClient, nil } return nil, nil, nil @@ -357,3 +322,67 @@ func (r *K8ssandraClusterReconciler) deleteK8ssandraConfigMaps( } return } + +func findDcForDeletionWithRemoteClient(ctx context.Context, + kcKey client.ObjectKey, + dcName string, + remoteClient client.Client) (*cassdcapi.CassandraDatacenter, error) { + + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + dcList := &cassdcapi.CassandraDatacenterList{} + + err := remoteClient.List(ctx, dcList, options) + if err != nil { + return nil, err + } + + for _, dc := range dcList.Items { + if dc.Name == dcName { + return &dc, nil + } + } + return nil, nil +} + +func finReaperForDeletionWithRemoteClient(ctx context.Context, + kcKey client.ObjectKey, + dcName, reaperName string, + remoteClient client.Client) (*reaperapi.Reaper, error) { + + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + reaperList := &reaperapi.ReaperList{} + + err := remoteClient.List(ctx, reaperList, options) + if err != nil { + return nil, err + } + for _, reaper := range reaperList.Items { + if reaper.Name == reaperName { + return &reaper, nil + } + } + return nil, nil +} + +func finStargateForDeletionWithRemoteClient(ctx context.Context, + kcKey client.ObjectKey, + dcName, stargateName string, + remoteClient client.Client) (*stargateapi.Stargate, error) { + + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + stargateList := &stargateapi.StargateList{} + + err := remoteClient.List(ctx, stargateList, options) + if err != nil { + return nil, err + } + for _, reaper := range stargateList.Items { + if reaper.Name == stargateName { + return &reaper, nil + } + } + return nil, nil +} From b16d82d63d5e27cbaeacddec7bfcecbb041b5923 Mon Sep 17 00:00:00 2001 From: Erik Merkle Date: Wed, 3 May 2023 13:12:36 -0500 Subject: [PATCH 2/4] Refactor deleteDc --- controllers/k8ssandra/cleanup.go | 147 +++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 48 deletions(-) diff --git a/controllers/k8ssandra/cleanup.go b/controllers/k8ssandra/cleanup.go index 34b8a5603..2ed116dcb 100644 --- a/controllers/k8ssandra/cleanup.go +++ b/controllers/k8ssandra/cleanup.go @@ -153,60 +153,22 @@ func (r *K8ssandraClusterReconciler) checkDcDeletion(ctx context.Context, kc *ap } func (r *K8ssandraClusterReconciler) deleteDc(ctx context.Context, kc *api.K8ssandraCluster, dcName string, logger logr.Logger) result.ReconcileResult { - kcKey := utils.GetKey(kc) + remoteClient := getFirstRemoteClient(r) - stargate, remoteClient, err := r.findStargateForDeletion(ctx, kcKey, dcName, nil) - if err != nil { - return result.Error(err) + // delete Stargate + if result := deleteStargate(r, ctx, kc, dcName, remoteClient, logger); result != nil { + return result } - if stargate != nil { - if err = remoteClient.Delete(ctx, stargate); err != nil && !errors.IsNotFound(err) { - return result.Error(fmt.Errorf("failed to delete Stargate for dc (%s): %v", dcName, err)) - } - logger.Info("Deleted Stargate", "Stargate", utils.GetKey(stargate)) + // delete Reaper + if result := deleteReaper(r, ctx, kc, dcName, remoteClient, logger); result != nil { + return result } - reaper, remoteClient, err := r.findReaperForDeletion(ctx, kcKey, dcName, remoteClient) - if err != nil { - return result.Error(err) + // delete the DC + if result := deleteDc(r, ctx, kc, dcName, remoteClient, logger); result != nil { + return result } - - if reaper != nil { - if err = remoteClient.Delete(ctx, reaper); err != nil && !errors.IsNotFound(err) { - return result.Error(fmt.Errorf("failed to delete Reaper for dc (%s): %v", dcName, err)) - } - logger.Info("Deleted Reaper", "Reaper", utils.GetKey(reaper)) - } - - dc, remoteClient, err := r.findDcForDeletion(ctx, kcKey, dcName, remoteClient) - if err != nil { - return result.Error(err) - } - - if dc != nil { - if dc.GetConditionStatus(cassdcapi.DatacenterDecommission) == corev1.ConditionTrue { - logger.Info("CassandraDatacenter decommissioning in progress", "CassandraDatacenter", utils.GetKey(dc)) - // There is no need to requeue here. Reconciliation will be trigger by updates made by cass-operator. - return result.Done() - } - - if !annotations.HasAnnotationWithValue(dc, cassdcapi.DecommissionOnDeleteAnnotation, "true") { - patch := client.MergeFrom(dc.DeepCopy()) - annotations.AddAnnotation(dc, cassdcapi.DecommissionOnDeleteAnnotation, "true") - if err = remoteClient.Patch(ctx, dc, patch); err != nil { - return result.Error(fmt.Errorf("failed to add %s annotation to dc: %v", cassdcapi.DecommissionOnDeleteAnnotation, err)) - } - } - - if err = remoteClient.Delete(ctx, dc); err != nil && !errors.IsNotFound(err) { - return result.Error(fmt.Errorf("failed to delete CassandraDatacenter (%s): %v", dcName, err)) - } - logger.Info("Deleted CassandraDatacenter", "CassandraDatacenter", utils.GetKey(dc)) - // There is no need to requeue here. Reconciliation will be trigger by updates made by cass-operator. - return result.Done() - } - delete(kc.Status.Datacenters, dcName) logger.Info("DC deletion finished", "DC", dcName) return result.Continue() @@ -386,3 +348,92 @@ func finStargateForDeletionWithRemoteClient(ctx context.Context, } return nil, nil } + +func getFirstRemoteClient(r *K8ssandraClusterReconciler) client.Client { + return r.ClientCache.GetAllClients()[0] +} + +func deleteStargate(r *K8ssandraClusterReconciler, + ctx context.Context, + kc *api.K8ssandraCluster, + dcName string, + remoteClient client.Client, + logger logr.Logger) result.ReconcileResult { + + kcKey := utils.GetKey(kc) + + stargate, remoteClient, err := r.findStargateForDeletion(ctx, kcKey, dcName, remoteClient) + if err != nil { + return result.Error(err) + } + + if stargate != nil { + if err = remoteClient.Delete(ctx, stargate); err != nil && !errors.IsNotFound(err) { + return result.Error(fmt.Errorf("failed to delete Stargate for dc (%s): %v", dcName, err)) + } + logger.Info("Deleted Stargate", "Stargate", utils.GetKey(stargate)) + } + return nil +} + +func deleteReaper(r *K8ssandraClusterReconciler, + ctx context.Context, + kc *api.K8ssandraCluster, + dcName string, + remoteClient client.Client, + logger logr.Logger) result.ReconcileResult { + + kcKey := utils.GetKey(kc) + + reaper, remoteClient, err := r.findReaperForDeletion(ctx, kcKey, dcName, remoteClient) + if err != nil { + return result.Error(err) + } + + if reaper != nil { + if err = remoteClient.Delete(ctx, reaper); err != nil && !errors.IsNotFound(err) { + return result.Error(fmt.Errorf("failed to delete Reaper for dc (%s): %v", dcName, err)) + } + logger.Info("Deleted Reaper", "Reaper", utils.GetKey(reaper)) + } + return nil +} + +func deleteDc(r *K8ssandraClusterReconciler, + ctx context.Context, + kc *api.K8ssandraCluster, + dcName string, + remoteClient client.Client, + logger logr.Logger) result.ReconcileResult { + + kcKey := utils.GetKey(kc) + + dc, remoteClient, err := r.findDcForDeletion(ctx, kcKey, dcName, remoteClient) + if err != nil { + return result.Error(err) + } + + if dc != nil { + if dc.GetConditionStatus(cassdcapi.DatacenterDecommission) == corev1.ConditionTrue { + logger.Info("CassandraDatacenter decommissioning in progress", "CassandraDatacenter", utils.GetKey(dc)) + // There is no need to requeue here. Reconciliation will be trigger by updates made by cass-operator. + return result.Done() + } + + if !annotations.HasAnnotationWithValue(dc, cassdcapi.DecommissionOnDeleteAnnotation, "true") { + patch := client.MergeFrom(dc.DeepCopy()) + annotations.AddAnnotation(dc, cassdcapi.DecommissionOnDeleteAnnotation, "true") + if err = remoteClient.Patch(ctx, dc, patch); err != nil { + return result.Error(fmt.Errorf("failed to add %s annotation to dc: %v", cassdcapi.DecommissionOnDeleteAnnotation, err)) + } + } + + if err = remoteClient.Delete(ctx, dc); err != nil && !errors.IsNotFound(err) { + return result.Error(fmt.Errorf("failed to delete CassandraDatacenter (%s): %v", dcName, err)) + } + logger.Info("Deleted CassandraDatacenter", "CassandraDatacenter", utils.GetKey(dc)) + // There is no need to requeue here. Reconciliation will be trigger by updates made by cass-operator. + return result.Done() + } + return nil +} From 2d7eaec5f8090aa6720b9b73e3897cdc44ce2538 Mon Sep 17 00:00:00 2001 From: Erik Merkle Date: Wed, 3 May 2023 13:45:41 -0500 Subject: [PATCH 3/4] simplify findForDelete functions --- controllers/k8ssandra/cleanup.go | 166 ++++++++++--------------------- 1 file changed, 50 insertions(+), 116 deletions(-) diff --git a/controllers/k8ssandra/cleanup.go b/controllers/k8ssandra/cleanup.go index 2ed116dcb..76f5b42ac 100644 --- a/controllers/k8ssandra/cleanup.go +++ b/controllers/k8ssandra/cleanup.go @@ -178,80 +178,78 @@ func (r *K8ssandraClusterReconciler) findStargateForDeletion( ctx context.Context, kcKey client.ObjectKey, dcName string, - remoteClient client.Client) (*stargateapi.Stargate, client.Client, error) { + remoteClient client.Client) (*stargateapi.Stargate, error) { - stargateName := kcKey.Name + "-" + dcName + "-stargate" if remoteClient == nil { - for _, remoteClient := range r.ClientCache.GetAllClients() { - stargate, err := finStargateForDeletionWithRemoteClient(ctx, kcKey, dcName, stargateName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) - } - return stargate, remoteClient, nil - } - } else { - stargate, err := finStargateForDeletionWithRemoteClient(ctx, kcKey, dcName, stargateName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) - } - - return stargate, remoteClient, nil + return nil, nil } + stargateName := kcKey.Name + "-" + dcName + "-stargate" + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + stargateList := &stargateapi.StargateList{} - return nil, nil, nil + err := remoteClient.List(ctx, stargateList, options) + if err != nil { + return nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) + } + for _, stargate := range stargateList.Items { + if stargate.Name == stargateName { + return &stargate, nil + } + } + return nil, nil } func (r *K8ssandraClusterReconciler) findReaperForDeletion( ctx context.Context, kcKey client.ObjectKey, dcName string, - remoteClient client.Client) (*reaperapi.Reaper, client.Client, error) { + remoteClient client.Client) (*reaperapi.Reaper, error) { - reaperName := kcKey.Name + "-" + dcName + "-reaper" if remoteClient == nil { - for _, remoteClient := range r.ClientCache.GetAllClients() { - reaper, err := finReaperForDeletionWithRemoteClient(ctx, kcKey, dcName, reaperName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) - } - return reaper, remoteClient, nil - } - } else { - reaper, err := finReaperForDeletionWithRemoteClient(ctx, kcKey, dcName, reaperName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) - } - - return reaper, remoteClient, nil + return nil, nil } + reaperName := kcKey.Name + "-" + dcName + "-reaper" + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + reaperList := &reaperapi.ReaperList{} - return nil, nil, nil + err := remoteClient.List(ctx, reaperList, options) + if err != nil { + return nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) + } + for _, reaper := range reaperList.Items { + if reaper.Name == reaperName { + return &reaper, nil + } + } + return nil, nil } func (r *K8ssandraClusterReconciler) findDcForDeletion( ctx context.Context, kcKey client.ObjectKey, dcName string, - remoteClient client.Client) (*cassdcapi.CassandraDatacenter, client.Client, error) { + remoteClient client.Client) (*cassdcapi.CassandraDatacenter, error) { if remoteClient == nil { - for _, remoteClient := range r.ClientCache.GetAllClients() { - dc, err := findDcForDeletionWithRemoteClient(ctx, kcKey, dcName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to CassandraDatacenter (%s) for DC (%s) deletion: %v", dcName, dcName, err) - } - return dc, remoteClient, nil - } - } else { - dc, err := findDcForDeletionWithRemoteClient(ctx, kcKey, dcName, remoteClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to find CassandraDatacenter (%s) for deletion: %v", dcName, err) - } + return nil, nil + } + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + dcList := &cassdcapi.CassandraDatacenterList{} - return dc, remoteClient, nil + err := remoteClient.List(ctx, dcList, options) + if err != nil { + return nil, fmt.Errorf("failed to find CassandraDatacenter (%s) for deletion: %v", dcName, err) } - return nil, nil, nil + for _, dc := range dcList.Items { + if dc.Name == dcName { + return &dc, nil + } + } + return nil, nil } func (r *K8ssandraClusterReconciler) deleteK8ssandraConfigMaps( @@ -285,70 +283,6 @@ func (r *K8ssandraClusterReconciler) deleteK8ssandraConfigMaps( return } -func findDcForDeletionWithRemoteClient(ctx context.Context, - kcKey client.ObjectKey, - dcName string, - remoteClient client.Client) (*cassdcapi.CassandraDatacenter, error) { - - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - dcList := &cassdcapi.CassandraDatacenterList{} - - err := remoteClient.List(ctx, dcList, options) - if err != nil { - return nil, err - } - - for _, dc := range dcList.Items { - if dc.Name == dcName { - return &dc, nil - } - } - return nil, nil -} - -func finReaperForDeletionWithRemoteClient(ctx context.Context, - kcKey client.ObjectKey, - dcName, reaperName string, - remoteClient client.Client) (*reaperapi.Reaper, error) { - - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - reaperList := &reaperapi.ReaperList{} - - err := remoteClient.List(ctx, reaperList, options) - if err != nil { - return nil, err - } - for _, reaper := range reaperList.Items { - if reaper.Name == reaperName { - return &reaper, nil - } - } - return nil, nil -} - -func finStargateForDeletionWithRemoteClient(ctx context.Context, - kcKey client.ObjectKey, - dcName, stargateName string, - remoteClient client.Client) (*stargateapi.Stargate, error) { - - selector := k8ssandralabels.PartOfLabels(kcKey) - options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} - stargateList := &stargateapi.StargateList{} - - err := remoteClient.List(ctx, stargateList, options) - if err != nil { - return nil, err - } - for _, reaper := range stargateList.Items { - if reaper.Name == stargateName { - return &reaper, nil - } - } - return nil, nil -} - func getFirstRemoteClient(r *K8ssandraClusterReconciler) client.Client { return r.ClientCache.GetAllClients()[0] } @@ -362,7 +296,7 @@ func deleteStargate(r *K8ssandraClusterReconciler, kcKey := utils.GetKey(kc) - stargate, remoteClient, err := r.findStargateForDeletion(ctx, kcKey, dcName, remoteClient) + stargate, err := r.findStargateForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { return result.Error(err) } @@ -385,7 +319,7 @@ func deleteReaper(r *K8ssandraClusterReconciler, kcKey := utils.GetKey(kc) - reaper, remoteClient, err := r.findReaperForDeletion(ctx, kcKey, dcName, remoteClient) + reaper, err := r.findReaperForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { return result.Error(err) } @@ -408,7 +342,7 @@ func deleteDc(r *K8ssandraClusterReconciler, kcKey := utils.GetKey(kc) - dc, remoteClient, err := r.findDcForDeletion(ctx, kcKey, dcName, remoteClient) + dc, err := r.findDcForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { return result.Error(err) } From 3c3c88453dd5f4585b635e8feeafa5e36eaa5f35 Mon Sep 17 00:00:00 2001 From: Erik Merkle Date: Thu, 4 May 2023 17:10:38 -0500 Subject: [PATCH 4/4] [fixup] fix remoteClient issues --- controllers/k8ssandra/cleanup.go | 103 +++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/controllers/k8ssandra/cleanup.go b/controllers/k8ssandra/cleanup.go index 76f5b42ac..7f24bad30 100644 --- a/controllers/k8ssandra/cleanup.go +++ b/controllers/k8ssandra/cleanup.go @@ -153,20 +153,19 @@ func (r *K8ssandraClusterReconciler) checkDcDeletion(ctx context.Context, kc *ap } func (r *K8ssandraClusterReconciler) deleteDc(ctx context.Context, kc *api.K8ssandraCluster, dcName string, logger logr.Logger) result.ReconcileResult { - remoteClient := getFirstRemoteClient(r) // delete Stargate - if result := deleteStargate(r, ctx, kc, dcName, remoteClient, logger); result != nil { + if result := r.deleteStargate(ctx, kc, dcName, logger); result != nil { return result } // delete Reaper - if result := deleteReaper(r, ctx, kc, dcName, remoteClient, logger); result != nil { + if result := r.deleteReaper(ctx, kc, dcName, logger); result != nil { return result } // delete the DC - if result := deleteDc(r, ctx, kc, dcName, remoteClient, logger); result != nil { + if result := r.deleteDatacenter(ctx, kc, dcName, logger); result != nil { return result } delete(kc.Status.Datacenters, dcName) @@ -283,18 +282,13 @@ func (r *K8ssandraClusterReconciler) deleteK8ssandraConfigMaps( return } -func getFirstRemoteClient(r *K8ssandraClusterReconciler) client.Client { - return r.ClientCache.GetAllClients()[0] -} - -func deleteStargate(r *K8ssandraClusterReconciler, - ctx context.Context, - kc *api.K8ssandraCluster, - dcName string, - remoteClient client.Client, - logger logr.Logger) result.ReconcileResult { +func (r *K8ssandraClusterReconciler) deleteStargate(ctx context.Context, kc *api.K8ssandraCluster, dcName string, logger logr.Logger) result.ReconcileResult { kcKey := utils.GetKey(kc) + remoteClient, err := r.findRemoteClientForStargate(ctx, kcKey, dcName) + if err != nil { + return result.Error(err) + } stargate, err := r.findStargateForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { @@ -310,14 +304,13 @@ func deleteStargate(r *K8ssandraClusterReconciler, return nil } -func deleteReaper(r *K8ssandraClusterReconciler, - ctx context.Context, - kc *api.K8ssandraCluster, - dcName string, - remoteClient client.Client, - logger logr.Logger) result.ReconcileResult { +func (r *K8ssandraClusterReconciler) deleteReaper(ctx context.Context, kc *api.K8ssandraCluster, dcName string, logger logr.Logger) result.ReconcileResult { kcKey := utils.GetKey(kc) + remoteClient, err := r.findRemoteClientForReaper(ctx, kcKey, dcName) + if err != nil { + return result.Error(err) + } reaper, err := r.findReaperForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { @@ -333,14 +326,13 @@ func deleteReaper(r *K8ssandraClusterReconciler, return nil } -func deleteDc(r *K8ssandraClusterReconciler, - ctx context.Context, - kc *api.K8ssandraCluster, - dcName string, - remoteClient client.Client, - logger logr.Logger) result.ReconcileResult { +func (r *K8ssandraClusterReconciler) deleteDatacenter(ctx context.Context, kc *api.K8ssandraCluster, dcName string, logger logr.Logger) result.ReconcileResult { kcKey := utils.GetKey(kc) + remoteClient, err := r.findRemoteClientForDc(ctx, kcKey, dcName) + if err != nil { + return result.Error(err) + } dc, err := r.findDcForDeletion(ctx, kcKey, dcName, remoteClient) if err != nil { @@ -371,3 +363,62 @@ func deleteDc(r *K8ssandraClusterReconciler, } return nil } + +func (r *K8ssandraClusterReconciler) findRemoteClientForReaper(ctx context.Context, kcKey client.ObjectKey, dcName string) (client.Client, error) { + reaperName := kcKey.Name + "-" + dcName + "-reaper" + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + reaperList := &reaperapi.ReaperList{} + + for _, remoteClient := range r.ClientCache.GetAllClients() { + err := remoteClient.List(ctx, reaperList, options) + if err != nil { + return nil, fmt.Errorf("failed to find Reaper (%s) for DC (%s) deletion: %v", reaperName, dcName, err) + } + for _, reaper := range reaperList.Items { + if reaper.Name == reaperName { + return remoteClient, nil + } + } + } + return nil, nil +} + +func (r *K8ssandraClusterReconciler) findRemoteClientForStargate(ctx context.Context, kcKey client.ObjectKey, dcName string) (client.Client, error) { + stargateName := kcKey.Name + "-" + dcName + "-stargate" + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + stargateList := &stargateapi.StargateList{} + + for _, remoteClient := range r.ClientCache.GetAllClients() { + err := remoteClient.List(ctx, stargateList, options) + if err != nil { + return nil, fmt.Errorf("failed to find Stargate (%s) for DC (%s) deletion: %v", stargateName, dcName, err) + } + for _, stargate := range stargateList.Items { + if stargate.Name == stargateName { + return remoteClient, nil + } + } + } + return nil, nil +} + +func (r *K8ssandraClusterReconciler) findRemoteClientForDc(ctx context.Context, kcKey client.ObjectKey, dcName string) (client.Client, error) { + selector := k8ssandralabels.PartOfLabels(kcKey) + options := &client.ListOptions{LabelSelector: labels.SelectorFromSet(selector)} + dcList := &cassdcapi.CassandraDatacenterList{} + + for _, remoteClient := range r.ClientCache.GetAllClients() { + err := remoteClient.List(ctx, dcList, options) + if err != nil { + return nil, fmt.Errorf("failed to find CassandraDatacenter (%s) for deletion: %v", dcName, err) + } + for _, reaper := range dcList.Items { + if reaper.Name == dcName { + return remoteClient, nil + } + } + } + return nil, nil +}