From 77880072573f4fc235cace82da1ee8c7a7385558 Mon Sep 17 00:00:00 2001 From: prsurve Date: Mon, 7 Oct 2024 12:46:42 +0530 Subject: [PATCH 1/6] Check for Kube Object Sync time before running failover for Discovered Apps Signed-off-by: prsurve --- ocs_ci/helpers/dr_helpers.py | 44 +++++++++++++++++++ ocs_ci/ocs/resources/drpc.py | 16 +++++++ ...t_failover_and_relocate_discovered_apps.py | 14 +++++- 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index 32a6c96a1b7..d3fef617986 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1715,3 +1715,47 @@ def verify_drpc_deletion(cmd, expected_output_lst): if expected_output not in drpc_out.stderr.decode(): return False return True + + +def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): + """ + Verifies that the lastKubeObjectProtectionTime for a given DRPC object is within the expected range. + + Args: + drpc_obj (obj): DRPC object + kubeoject_sync_interval (int): The KubeObject sync interval in minutes + + Returns: + str: Current lastKubeObjectProtectionTime + + Raises: + AssertionError: If the lastKubeObjectProtectionTime is outside the expected range + (greater than or equal to three times the scheduling interval) + + """ + restore_index = config.cur_index + config.switch_acm_ctx() + last_kubeobject_protection_time = drpc_obj.get_last_kubeobject_protection_time() + if not last_kubeobject_protection_time: + assert last_kubeobject_protection_time, ( + "There is no lastKubeObjectProtectionTime. " + "Verify that certificates are included correctly in the Ramen Hub configuration map." + ) + # Verify lastGroupSyncTime + time_format = "%Y-%m-%dT%H:%M:%SZ" + last_kubeobject_protection_time_formatted = datetime.strptime( + last_kubeobject_protection_time, time_format + ) + current_time = datetime.strptime( + datetime.utcnow().strftime(time_format), time_format + ) + time_since_last_sync = ( + current_time - last_kubeobject_protection_time_formatted + ).total_seconds() / 60 + logger.info(f"Time in minutes since the last sync {time_since_last_sync}") + assert ( + time_since_last_sync < 3 * kubeoject_sync_interval + ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" + logger.info("Verified lastKubeObjectProtectionTime value within expected range") + config.switch_ctx(restore_index) + return last_kubeobject_protection_time diff --git a/ocs_ci/ocs/resources/drpc.py b/ocs_ci/ocs/resources/drpc.py index bad71dc4d71..6972271f663 100644 --- a/ocs_ci/ocs/resources/drpc.py +++ b/ocs_ci/ocs/resources/drpc.py @@ -101,6 +101,22 @@ def get_last_group_sync_time(self): logger.info(f"Current lastGroupSyncTime is {last_group_sync_time}.") return last_group_sync_time + def get_last_kubeobject_protection_time(self): + """ + Fetch lastKubeObjectProtectionTime from DRPC + + Returns: + str: lastKubeObjectProtectionTime + + """ + last_kubeobject_protection_time = ( + self.get().get("status").get("lastKubeObjectProtectionTime") + ) + logger.info( + f"Current lastKubeObjectProtectionTime is {last_kubeobject_protection_time}." + ) + return last_kubeobject_protection_time + def get_drpc_name(namespace, switch_ctx=None): """ diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index 3cd4b4b8b25..65dfe483fd8 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -6,7 +6,7 @@ from ocs_ci.framework.testlib import acceptance, tier1 from ocs_ci.framework.pytest_customization.marks import rdr, turquoise_squad from ocs_ci.helpers import dr_helpers - +from ocs_ci.ocs.resources.drpc import DRPC logger = logging.getLogger(__name__) @@ -45,10 +45,16 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload scheduling_interval = dr_helpers.get_scheduling_interval( rdr_workload.workload_namespace, discovered_apps=True ) + drpc_obj = DRPC(namespace=rdr_workload.workload_namespace) wait_time = 2 * scheduling_interval # Time in minutes logger.info(f"Waiting for {wait_time} minutes to run IOs") sleep(wait_time * 60) + logger.info("Checking for lastKubeObjectProtectionTime") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval + ) + dr_helpers.failover( failover_cluster=secondary_cluster_name, namespace=rdr_workload.workload_namespace, @@ -87,11 +93,17 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload scheduling_interval = dr_helpers.get_scheduling_interval( rdr_workload.workload_namespace, discovered_apps=True ) + logger.info("Running Relocate Steps") wait_time = 2 * scheduling_interval # Time in minutes logger.info(f"Waiting for {wait_time} minutes to run IOs") sleep(wait_time * 60) + logger.info("Checking for lastKubeObjectProtectionTime") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval + ) + dr_helpers.relocate( preferred_cluster=secondary_cluster_name, namespace=rdr_workload.workload_namespace, From fc091c03a22ae2735e84147e4b2b3e793d6adbf2 Mon Sep 17 00:00:00 2001 From: prsurve Date: Mon, 7 Oct 2024 16:19:30 +0530 Subject: [PATCH 2/6] Check for Kube Object Sync time before running failover for Discovered Apps Signed-off-by: prsurve --- ocs_ci/helpers/dr_helpers.py | 2 +- ocs_ci/ocs/dr/dr_workload.py | 3 ++- .../test_failover_and_relocate_discovered_apps.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index d3fef617986..2a3a035de63 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1754,7 +1754,7 @@ def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): ).total_seconds() / 60 logger.info(f"Time in minutes since the last sync {time_since_last_sync}") assert ( - time_since_last_sync < 3 * kubeoject_sync_interval + time_since_last_sync < 2 * kubeoject_sync_interval ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" logger.info("Verified lastKubeObjectProtectionTime value within expected range") config.switch_ctx(restore_index) diff --git a/ocs_ci/ocs/dr/dr_workload.py b/ocs_ci/ocs/dr/dr_workload.py index 42162d66732..b62d758c693 100644 --- a/ocs_ci/ocs/dr/dr_workload.py +++ b/ocs_ci/ocs/dr/dr_workload.py @@ -1076,7 +1076,8 @@ def __init__(self, **kwargs): self.discovered_apps_placement_name = kwargs.get("workload_placement_name") self.drpc_yaml_file = os.path.join(constants.DRPC_PATH) self.placement_yaml_file = os.path.join(constants.PLACEMENT_PATH) - self.kubeobject_capture_interval = f"{generate_kubeobject_capture_interval()}m" + self.kubeobject_capture_interval_int = generate_kubeobject_capture_interval() + self.kubeobject_capture_interval = f"{self.kubeobject_capture_interval_int}m" self.protection_type = kwargs.get("protection_type") self.target_clone_dir = config.ENV_DATA.get( "target_clone_dir", constants.DR_WORKLOAD_REPO_BASE_DIR diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index 65dfe483fd8..9c9c08f0838 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -6,6 +6,7 @@ from ocs_ci.framework.testlib import acceptance, tier1 from ocs_ci.framework.pytest_customization.marks import rdr, turquoise_squad from ocs_ci.helpers import dr_helpers +from ocs_ci.ocs import constants from ocs_ci.ocs.resources.drpc import DRPC logger = logging.getLogger(__name__) @@ -45,14 +46,14 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload scheduling_interval = dr_helpers.get_scheduling_interval( rdr_workload.workload_namespace, discovered_apps=True ) - drpc_obj = DRPC(namespace=rdr_workload.workload_namespace) + drpc_obj = DRPC(namespace=constants.DR_OPS_NAMESAPCE) wait_time = 2 * scheduling_interval # Time in minutes logger.info(f"Waiting for {wait_time} minutes to run IOs") sleep(wait_time * 60) logger.info("Checking for lastKubeObjectProtectionTime") dr_helpers.verify_last_kubeobject_protection_time( - drpc_obj, rdr_workload.kubeobject_capture_interval + drpc_obj, rdr_workload.kubeobject_capture_interval_int ) dr_helpers.failover( From 35807dd7eac262eec2ed5cd6f95d1a5acd89d47f Mon Sep 17 00:00:00 2001 From: prsurve Date: Mon, 7 Oct 2024 18:19:22 +0530 Subject: [PATCH 3/6] Update param name Signed-off-by: prsurve --- ocs_ci/helpers/dr_helpers.py | 2 +- .../regional-dr/test_failover_and_relocate_discovered_apps.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index 2a3a035de63..87aaf953e17 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1752,7 +1752,7 @@ def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): time_since_last_sync = ( current_time - last_kubeobject_protection_time_formatted ).total_seconds() / 60 - logger.info(f"Time in minutes since the last sync {time_since_last_sync}") + logger.info(f"Time in minutes since the last Kube Object sync {time_since_last_sync}") assert ( time_since_last_sync < 2 * kubeoject_sync_interval ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index 9c9c08f0838..51cdb9f6dd8 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -102,7 +102,7 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload logger.info("Checking for lastKubeObjectProtectionTime") dr_helpers.verify_last_kubeobject_protection_time( - drpc_obj, rdr_workload.kubeobject_capture_interval + drpc_obj, rdr_workload.kubeobject_capture_interval_int ) dr_helpers.relocate( From 2928f7561f68acb9b38ecdf87b7cc29aa908bcec Mon Sep 17 00:00:00 2001 From: prsurve Date: Mon, 7 Oct 2024 18:28:54 +0530 Subject: [PATCH 4/6] Fix tox Signed-off-by: prsurve --- ocs_ci/helpers/dr_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index 87aaf953e17..70a260d08df 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1752,7 +1752,9 @@ def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): time_since_last_sync = ( current_time - last_kubeobject_protection_time_formatted ).total_seconds() / 60 - logger.info(f"Time in minutes since the last Kube Object sync {time_since_last_sync}") + logger.info( + f"Time in minutes since the last Kube Object sync {time_since_last_sync}" + ) assert ( time_since_last_sync < 2 * kubeoject_sync_interval ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" From 48f939d895d10c2a12c6f6270cd755cd74d517a9 Mon Sep 17 00:00:00 2001 From: prsurve Date: Thu, 17 Oct 2024 19:11:11 +0530 Subject: [PATCH 5/6] Add Skip to older versions Signed-off-by: prsurve --- ocs_ci/helpers/dr_helpers.py | 6 +++--- .../test_failover_and_relocate_discovered_apps.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index 70a260d08df..c5aa7b2415a 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1717,13 +1717,13 @@ def verify_drpc_deletion(cmd, expected_output_lst): return True -def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): +def verify_last_kubeobject_protection_time(drpc_obj, kubeobject_sync_interval): """ Verifies that the lastKubeObjectProtectionTime for a given DRPC object is within the expected range. Args: drpc_obj (obj): DRPC object - kubeoject_sync_interval (int): The KubeObject sync interval in minutes + kubeobject_sync_interval (int): The KubeObject sync interval in minutes Returns: str: Current lastKubeObjectProtectionTime @@ -1756,7 +1756,7 @@ def verify_last_kubeobject_protection_time(drpc_obj, kubeoject_sync_interval): f"Time in minutes since the last Kube Object sync {time_since_last_sync}" ) assert ( - time_since_last_sync < 2 * kubeoject_sync_interval + time_since_last_sync < 2 * kubeobject_sync_interval ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" logger.info("Verified lastKubeObjectProtectionTime value within expected range") config.switch_ctx(restore_index) diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index 51cdb9f6dd8..f5a593647d1 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -3,7 +3,7 @@ from ocs_ci.framework import config -from ocs_ci.framework.testlib import acceptance, tier1 +from ocs_ci.framework.testlib import acceptance, tier1, skipif_ocs_version from ocs_ci.framework.pytest_customization.marks import rdr, turquoise_squad from ocs_ci.helpers import dr_helpers from ocs_ci.ocs import constants @@ -16,6 +16,7 @@ @acceptance @tier1 @turquoise_squad +@skipif_ocs_version("<4.16") class TestFailoverAndRelocateWithDiscoveredApps: """ Test Failover and Relocate with Discovered Apps From 8a3a34653546c308f1c01e5c1d0261d0cf800d2d Mon Sep 17 00:00:00 2001 From: prsurve Date: Mon, 21 Oct 2024 14:52:12 +0530 Subject: [PATCH 6/6] Check for lastKubeObjectProtectionTime post relocate Signed-off-by: prsurve --- .../test_failover_and_relocate_discovered_apps.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index f5a593647d1..11b805b1368 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -115,4 +115,9 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload workload_instance=rdr_workload, ) + logger.info("Checking for lastKubeObjectProtectionTime post Relocate Operation") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval_int + ) + # TODO: Add data integrity checks