From 5e8bca8cad42880b1ed8cbd8246cd189e77e6a54 Mon Sep 17 00:00:00 2001
From: Benamar Mekhissi <bmekhiss@ibm.com>
Date: Fri, 22 Dec 2023 11:17:33 -0500
Subject: [PATCH] Check access to VRG on a MC before deleting the MW

Signed-off-by: Benamar Mekhissi <bmekhiss@ibm.com>
---
 controllers/drcluster_mmode.go               |  2 +-
 controllers/drplacementcontrol.go            | 30 ++++++++++++++++++--
 controllers/drplacementcontrol_controller.go |  7 +++--
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/controllers/drcluster_mmode.go b/controllers/drcluster_mmode.go
index 41defed879..57d1b8d19e 100644
--- a/controllers/drcluster_mmode.go
+++ b/controllers/drcluster_mmode.go
@@ -62,7 +62,7 @@ func (u *drclusterInstance) mModeActivationsRequired() (map[string]ramen.Storage
 		vrgs, err := u.getVRGs(drpcCollection)
 		if err != nil {
 			u.log.Info("Failed to get VRGs for DRPC that is failing over",
-				"DRPCCommonName", drpcCollection.drpc.GetName(),
+				"DRPCName", drpcCollection.drpc.GetName(),
 				"DRPCNamespace", drpcCollection.drpc.GetNamespace())
 
 			u.requeue = true
diff --git a/controllers/drplacementcontrol.go b/controllers/drplacementcontrol.go
index 1a16c7f33d..e0eac58079 100644
--- a/controllers/drplacementcontrol.go
+++ b/controllers/drplacementcontrol.go
@@ -559,7 +559,7 @@ func requiresRegionalFailoverPrerequisites(
 	ctx context.Context,
 	apiReader client.Reader,
 	s3ProfileNames []string,
-	DRPCCommonName string,
+	drpcName string,
 	vrgNamespace string,
 	vrgs map[string]*rmn.VolumeReplicationGroup,
 	failoverCluster string,
@@ -573,7 +573,7 @@ func requiresRegionalFailoverPrerequisites(
 
 	vrg := getLastKnownPrimaryVRG(vrgs, failoverCluster)
 	if vrg == nil {
-		vrg = GetLastKnownVRGPrimaryFromS3(ctx, apiReader, s3ProfileNames, DRPCCommonName, vrgNamespace, objectStoreGetter, log)
+		vrg = GetLastKnownVRGPrimaryFromS3(ctx, apiReader, s3ProfileNames, drpcName, vrgNamespace, objectStoreGetter, log)
 		if vrg == nil {
 			// TODO: Is this an error, should we ensure at least one VRG is found in the edge cases?
 			// Potentially missing VRG and so stop failover? How to recover in that case?
@@ -1340,7 +1340,12 @@ func (d *DRPCInstance) cleanupSecondaries(skipCluster string) (bool, error) {
 		}
 
 		// If VRG hasn't been deleted, then make sure that the MW for it is deleted and
-		// return and wait
+		// return and wait, but first make sure that the cluster is accessible
+		if err := checkAccessToVRGOnCluster(d.reconciler.MCVGetter, d.instance.GetName(), d.instance.GetNamespace(),
+			d.vrgNamespace, clusterName); err != nil {
+			return false, err
+		}
+
 		mwDeleted, err := d.ensureVRGManifestWorkOnClusterDeleted(clusterName)
 		if err != nil {
 			return false, err
@@ -1380,6 +1385,25 @@ func (d *DRPCInstance) cleanupSecondaries(skipCluster string) (bool, error) {
 	return true, nil
 }
 
+func checkAccessToVRGOnCluster(mcvGetter rmnutil.ManagedClusterViewGetter,
+	name, drpcNamespace, vrgNamespace, clusterName string,
+) error {
+	annotations := make(map[string]string)
+
+	annotations[DRPCNameAnnotation] = name
+	annotations[DRPCNamespaceAnnotation] = drpcNamespace
+
+	_, err := mcvGetter.GetVRGFromManagedCluster(name,
+		vrgNamespace, clusterName, annotations)
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (d *DRPCInstance) updateUserPlacementRule(homeCluster, reason string) error {
 	d.log.Info(fmt.Sprintf("Updating user Placement %s homeCluster %s",
 		d.userPlacement.GetName(), homeCluster))
diff --git a/controllers/drplacementcontrol_controller.go b/controllers/drplacementcontrol_controller.go
index bd6644f0f7..f4f158fd52 100644
--- a/controllers/drplacementcontrol_controller.go
+++ b/controllers/drplacementcontrol_controller.go
@@ -2166,6 +2166,8 @@ func (r *DRPlacementControlReconciler) determineDRPCState(
 	dstCluster string,
 	log logr.Logger,
 ) (Progress, error) {
+	log.Info("Rebuild DRPC state")
+
 	vrgNamespace, err := selectVRGNamespace(r.Client, log, drpc, placementObj)
 	if err != nil {
 		log.Info("Failed to select VRG namespace")
@@ -2286,8 +2288,9 @@ func (r *DRPlacementControlReconciler) determineDRPCState(
 
 		for k, v := range vrgs {
 			clusterName, vrg = k, v
-
-			break
+			if vrg.Spec.ReplicationState == rmn.Primary {
+				break
+			}
 		}
 
 		if drpc.Spec.Action == rmn.DRAction(vrg.Spec.Action) {