From 05954f656b33d2f70e8c7ea658a404fa8450be83 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Tue, 14 Nov 2023 19:29:13 +0200 Subject: [PATCH 1/2] Deploy recipe CRD in all DR environments Recently the recipe CRD[1] became required if kube object protection is enabled, so we must have it in the test environment unless we disable the feature. Disabling the feature is not a good idea since we will want to be able to test imperative apps or declerative apps that need recipes (e.g. kubvirt with DataVolumeTemplate). [1] https://github.com/RamenDR/recipe/blob/main/config/crd/bases/ramendr.openshift.io_recipes.yaml Signed-off-by: Nir Soffer --- test/addons/recipe/start | 22 +++++++++++++++++++++ test/envs/regional-dr-external.yaml.example | 1 + test/envs/regional-dr-hubless.yaml | 1 + test/envs/regional-dr-kubevirt.yaml | 1 + test/envs/regional-dr.yaml | 1 + 5 files changed, 26 insertions(+) create mode 100755 test/addons/recipe/start diff --git a/test/addons/recipe/start b/test/addons/recipe/start new file mode 100755 index 000000000..6a8cc0f35 --- /dev/null +++ b/test/addons/recipe/start @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +from drenv import kubectl + +if len(sys.argv) != 2: + sys.exit(f"Usage: {sys.argv[0]} cluster") + +os.chdir(os.path.dirname(__file__)) +cluster = sys.argv[1] + +print("Deploying recipe crd") +kubectl.apply( + "--kustomize", + "https://github.com/RamenDR/recipe.git/config/crd?ref=main&timeout=120s", + context=cluster, +) diff --git a/test/envs/regional-dr-external.yaml.example b/test/envs/regional-dr-external.yaml.example index 1fb9395cc..a928b8ff6 100644 --- a/test/envs/regional-dr-external.yaml.example +++ b/test/envs/regional-dr-external.yaml.example @@ -20,6 +20,7 @@ templates: - addons: - name: ocm-cluster args: ["$name", "hub"] + - name: recipe - addons: - name: cert-manager - name: csi-addons diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index 6bb76bc9c..490ff7b18 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -33,6 +33,7 @@ templates: - name: olm - name: minio - name: velero + - name: recipe profiles: - name: "dr1" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index 980c3af64..c6527fa3b 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -38,6 +38,7 @@ templates: - name: ocm-cluster args: ["$name", "hub"] - name: cdi + - name: recipe - addons: - name: csi-addons - name: olm diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 0f7e208b3..f42fafe7f 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -31,6 +31,7 @@ templates: - addons: - name: ocm-cluster args: ["$name", "hub"] + - name: recipe - addons: - name: csi-addons - name: olm From 07b270c14b508f10dd0027a62341f620ca741243 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 22 Nov 2023 21:38:53 +0200 Subject: [PATCH 2/2] Deploy managed clusters s3 secrets using ramen In OpenShift we deploy 2 s3 secrets (one per s3 store) on the hub, and the secrets are propagated to the managed clusters using the policy framework. In ramenctl we deploy the secrets directly to the managed clusters. This is much simpler and more reliable, but it bypass the ramen code we want to test, hiding issues in the real code path. Change ramenctl to deploy the secrets in the same way as in OpenShift: - Use 2 secrets, one per cluster s3 store - Deploy the secrets only on the hub - Wait until the secrets are propagated to the managed clusters by ramen. With this issues in ramen related code or OCM will break ramenctl early. Hopefully this will help to detect regressions before they reach QE or released in OpenShift. Example run: $ ramenctl config $env 2023-11-22 22:05:19,546 INFO [ramenctl] Starting config 2023-11-22 22:05:19,812 INFO [ramenctl] Waiting until ramen-hub-operator is rolled out 2023-11-22 22:05:19,889 INFO [ramenctl] Creating ramen s3 secrets in cluster 'hub' 2023-11-22 22:05:20,428 INFO [ramenctl] Updating ramen config map in cluster 'hub' 2023-11-22 22:05:20,716 INFO [ramenctl] Creating dr-clusters for regional-dr 2023-11-22 22:05:20,988 INFO [ramenctl] Creating dr-policy for regional-dr 2023-11-22 22:05:21,220 INFO [ramenctl] Waiting until s3 secrets are propagated to managed clusters 2023-11-22 22:05:22,800 INFO [ramenctl] Waiting until DRClusters report phase 2023-11-22 22:05:22,941 INFO [ramenctl] Waiting until DRClusters phase is available 2023-11-22 22:05:23,206 INFO [ramenctl] Waiting until DRPolicy is validated 2023-11-22 22:05:23,361 INFO [ramenctl] Finished config in 3.82 seconds Signed-off-by: Nir Soffer --- ramenctl/ramenctl/config.py | 48 +++++++++++++++---- ramenctl/ramenctl/resources/configmap.yaml | 4 +- .../ramenctl/resources/ramen-s3-secret.yaml | 2 +- ramenctl/ramenctl/unconfig.py | 26 ++++++---- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index 7c5ef4fff..f3b0b2f91 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -21,7 +21,7 @@ def register(commands): def run(args): env = command.env_info(args) - s3_secret = generate_ramen_s3_secret(args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: @@ -29,19 +29,19 @@ def run(args): wait_for_ramen_hub_operator(env["hub"], args) - create_ramen_s3_secret(env["hub"], s3_secret) - for cluster in env["clusters"]: - create_cloud_credentials_secret(cluster, cloud_secret) + create_ramen_s3_secrets(env["hub"], s3_secrets) + create_ramen_config_map(env["hub"], hub_cm) create_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) + wait_for_secret_propagation(env["hub"], env["clusters"], args) wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) for cluster in env["clusters"]: - create_ramen_s3_secret(cluster, s3_secret) + create_ramen_s3_secrets(cluster, s3_secrets) create_cloud_credentials_secret(cluster, cloud_secret) create_ramen_config_map(cluster, dr_cluster_cm) @@ -58,14 +58,18 @@ def wait_for_ramen_hub_operator(hub, args): ) -def generate_ramen_s3_secret(args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - return template.substitute(namespace=args.ramen_namespace) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] -def create_ramen_s3_secret(cluster, yaml): - command.info("Creating ramen s3 secret in cluster '%s'", cluster) - kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) +def create_ramen_s3_secrets(cluster, secrets): + command.info("Creating ramen s3 secrets in cluster '%s'", cluster) + for secret in secrets: + kubectl.apply("--filename=-", input=secret, context=cluster, log=command.debug) def generate_cloud_credentials_secret(cluster, args): @@ -111,6 +115,30 @@ def create_hub_dr_resources(hub, clusters, topology): kubectl.apply("--filename=-", input=yaml, context=hub, log=command.debug) +def wait_for_secret_propagation(hub, clusters, args): + command.info("Waiting until s3 secrets are propagated to managed clusters") + for cluster in clusters: + policy = f"{args.ramen_namespace}.ramen-s3-secret-{cluster}" + command.debug("Waiting until policy '%s' reports status", policy) + drenv.wait_for( + f"policy/{policy}", + output="jsonpath={.status}", + namespace=cluster, + timeout=30, + profile=hub, + log=command.debug, + ) + command.debug("Waiting until policy %s is compliant", policy) + kubectl.wait( + f"policy/{policy}", + "--for=jsonpath={.status.compliant}=Compliant", + "--timeout=30s", + f"--namespace={cluster}", + context=hub, + log=command.debug, + ) + + def wait_for_dr_clusters(hub, clusters, args): command.info("Waiting until DRClusters report phase") for name in clusters: diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index d10b180ee..c82136f1f 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -40,7 +40,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster1 s3Region: us-west-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster1 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud @@ -50,7 +50,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster2 s3Region: us-east-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster2 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud diff --git a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml index e09c01961..88f49cf6b 100644 --- a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml +++ b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster namespace: $namespace stringData: AWS_ACCESS_KEY_ID: minio diff --git a/ramenctl/ramenctl/unconfig.py b/ramenctl/ramenctl/unconfig.py index 82c3714cf..b0abcc671 100644 --- a/ramenctl/ramenctl/unconfig.py +++ b/ramenctl/ramenctl/unconfig.py @@ -24,11 +24,11 @@ def run(args): if env["hub"]: delete_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) - delete_s3_secret([env["hub"]], args) - delete_cloud_credentials(env["clusters"], args) - else: - delete_s3_secret(env["clusters"], args) - delete_cloud_credentials(env["clusters"], args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) + delete_s3_secrets(env["hub"], s3_secrets) + + # TODO: Should be removed by ramen. + delete_cloud_credentials(env["clusters"], args) def delete_hub_dr_resources(hub, clusters, topology): @@ -46,15 +46,21 @@ def delete_hub_dr_resources(hub, clusters, topology): ) -def delete_s3_secret(clusters, args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - yaml = template.substitute(namespace=args.ramen_namespace) - for cluster in clusters: - command.info("Deleting s3 secret in cluster '%s'", cluster) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] + + +def delete_s3_secrets(cluster, secrets): + command.info("Deleting s3 secrets in cluster '%s'", cluster) + for secret in secrets: kubectl.delete( "--filename=-", "--ignore-not-found", - input=yaml, + input=secret, context=cluster, log=command.debug, )