From 1829ad850d32ab5d0dfaf832d567e781a954af29 Mon Sep 17 00:00:00 2001
From: prsurve <prsurve@redhat.com>
Date: Mon, 7 Oct 2024 12:46:42 +0530
Subject: [PATCH] Check for Kube Object Sync time before running failover for
 Discovered Apps

Signed-off-by: prsurve <prsurve@redhat.com>
---
 ocs_ci/helpers/dr_helpers.py                  | 44 +++++++++++++++++++
 ocs_ci/ocs/resources/drpc.py                  | 16 +++++++
 ...t_failover_and_relocate_discovered_apps.py | 14 +++++-
 3 files changed, 73 insertions(+), 1 deletion(-)

diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py
index d0008fabce5..5f0cbdaa29b 100644
--- a/ocs_ci/helpers/dr_helpers.py
+++ b/ocs_ci/helpers/dr_helpers.py
@@ -1661,3 +1661,47 @@ def generate_kubeobject_capture_interval():
         return 5
     else:
         return capture_interval
+
+
+def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval):
+    """
+    Verifies that the lastKubeObjectProtectionTime for a given DRPC object is within the expected range.
+
+    Args:
+        drpc_obj (obj): DRPC object
+        kubeoject_sync_interval (int): The KubeObject sync interval in minutes
+
+    Returns:
+        str: Current lastKubeObjectProtectionTime
+
+    Raises:
+        AssertionError: If the lastKubeObjectProtectionTime is outside the expected range
+            (greater than or equal to three times the scheduling interval)
+
+    """
+    restore_index = config.cur_index
+    config.switch_acm_ctx()
+    last_kubeobject_protection_time = drpc_obj.get_last_kubeobject_protection_time()
+    if not last_kubeobject_protection_time:
+        assert last_kubeobject_protection_time, (
+            "There is no lastKubeObjectProtectionTime. "
+            "Verify that certificates are included correctly in the Ramen Hub configuration map."
+        )
+    # Verify lastGroupSyncTime
+    time_format = "%Y-%m-%dT%H:%M:%SZ"
+    last_kubeobject_protection_time_formatted = datetime.strptime(
+        last_kubeobject_protection_time, time_format
+    )
+    current_time = datetime.strptime(
+        datetime.utcnow().strftime(time_format), time_format
+    )
+    time_since_last_sync = (
+        current_time - last_kubeobject_protection_time_formatted
+    ).total_seconds() / 60
+    logger.info(f"Time in minutes since the last sync {time_since_last_sync}")
+    assert (
+        time_since_last_sync < 3 * kubeoject_sync_interval
+    ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval"
+    logger.info("Verified lastKubeObjectProtectionTime value within expected range")
+    config.switch_ctx(restore_index)
+    return last_kubeobject_protection_time
diff --git a/ocs_ci/ocs/resources/drpc.py b/ocs_ci/ocs/resources/drpc.py
index bad71dc4d71..6972271f663 100644
--- a/ocs_ci/ocs/resources/drpc.py
+++ b/ocs_ci/ocs/resources/drpc.py
@@ -101,6 +101,22 @@ def get_last_group_sync_time(self):
         logger.info(f"Current lastGroupSyncTime is {last_group_sync_time}.")
         return last_group_sync_time
 
+    def get_last_kubeobject_protection_time(self):
+        """
+        Fetch lastKubeObjectProtectionTime from DRPC
+
+        Returns:
+            str: lastKubeObjectProtectionTime
+
+        """
+        last_kubeobject_protection_time = (
+            self.get().get("status").get("lastKubeObjectProtectionTime")
+        )
+        logger.info(
+            f"Current lastKubeObjectProtectionTime is {last_kubeobject_protection_time}."
+        )
+        return last_kubeobject_protection_time
+
 
 def get_drpc_name(namespace, switch_ctx=None):
     """
diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py
index ae9dd32a5f3..1413e5497a2 100644
--- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py
+++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py
@@ -6,7 +6,7 @@
 from ocs_ci.framework.testlib import acceptance, tier1
 from ocs_ci.framework.pytest_customization.marks import turquoise_squad
 from ocs_ci.helpers import dr_helpers
-
+from ocs_ci.ocs.resources.drpc import DRPC
 
 logger = logging.getLogger(__name__)
 
@@ -44,10 +44,16 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload
         scheduling_interval = dr_helpers.get_scheduling_interval(
             rdr_workload.workload_namespace, discovered_apps=True
         )
+        drpc_obj = DRPC(namespace=rdr_workload.workload_namespace)
         wait_time = 2 * scheduling_interval  # Time in minutes
         logger.info(f"Waiting for {wait_time} minutes to run IOs")
         sleep(wait_time * 60)
 
+        logger.info("Checking for lastKubeObjectProtectionTime")
+        dr_helpers.verify_last_kubeobject_protection_time(
+            drpc_obj, rdr_workload.kubeobject_capture_interval
+        )
+
         dr_helpers.failover(
             failover_cluster=secondary_cluster_name,
             namespace=rdr_workload.workload_namespace,
@@ -86,11 +92,17 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload
         scheduling_interval = dr_helpers.get_scheduling_interval(
             rdr_workload.workload_namespace, discovered_apps=True
         )
+
         logger.info("Running Relocate Steps")
         wait_time = 2 * scheduling_interval  # Time in minutes
         logger.info(f"Waiting for {wait_time} minutes to run IOs")
         sleep(wait_time * 60)
 
+        logger.info("Checking for lastKubeObjectProtectionTime")
+        dr_helpers.verify_last_kubeobject_protection_time(
+            drpc_obj, rdr_workload.kubeobject_capture_interval
+        )
+
         dr_helpers.relocate(
             preferred_cluster=secondary_cluster_name,
             namespace=rdr_workload.workload_namespace,