diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index 7c5ef4fff..f3b0b2f91 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -21,7 +21,7 @@ def register(commands): def run(args): env = command.env_info(args) - s3_secret = generate_ramen_s3_secret(args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: @@ -29,19 +29,19 @@ def run(args): wait_for_ramen_hub_operator(env["hub"], args) - create_ramen_s3_secret(env["hub"], s3_secret) - for cluster in env["clusters"]: - create_cloud_credentials_secret(cluster, cloud_secret) + create_ramen_s3_secrets(env["hub"], s3_secrets) + create_ramen_config_map(env["hub"], hub_cm) create_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) + wait_for_secret_propagation(env["hub"], env["clusters"], args) wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) for cluster in env["clusters"]: - create_ramen_s3_secret(cluster, s3_secret) + create_ramen_s3_secrets(cluster, s3_secrets) create_cloud_credentials_secret(cluster, cloud_secret) create_ramen_config_map(cluster, dr_cluster_cm) @@ -58,14 +58,18 @@ def wait_for_ramen_hub_operator(hub, args): ) -def generate_ramen_s3_secret(args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - return template.substitute(namespace=args.ramen_namespace) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] -def create_ramen_s3_secret(cluster, yaml): - command.info("Creating ramen s3 secret in cluster '%s'", cluster) - kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) +def create_ramen_s3_secrets(cluster, secrets): + command.info("Creating ramen s3 secrets in cluster '%s'", cluster) + for secret in secrets: + kubectl.apply("--filename=-", input=secret, context=cluster, log=command.debug) def generate_cloud_credentials_secret(cluster, args): @@ -111,6 +115,30 @@ def create_hub_dr_resources(hub, clusters, topology): kubectl.apply("--filename=-", input=yaml, context=hub, log=command.debug) +def wait_for_secret_propagation(hub, clusters, args): + command.info("Waiting until s3 secrets are propagated to managed clusters") + for cluster in clusters: + policy = f"{args.ramen_namespace}.ramen-s3-secret-{cluster}" + command.debug("Waiting until policy '%s' reports status", policy) + drenv.wait_for( + f"policy/{policy}", + output="jsonpath={.status}", + namespace=cluster, + timeout=30, + profile=hub, + log=command.debug, + ) + command.debug("Waiting until policy %s is compliant", policy) + kubectl.wait( + f"policy/{policy}", + "--for=jsonpath={.status.compliant}=Compliant", + "--timeout=30s", + f"--namespace={cluster}", + context=hub, + log=command.debug, + ) + + def wait_for_dr_clusters(hub, clusters, args): command.info("Waiting until DRClusters report phase") for name in clusters: diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index d10b180ee..c82136f1f 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -40,7 +40,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster1 s3Region: us-west-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster1 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud @@ -50,7 +50,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster2 s3Region: us-east-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster2 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud diff --git a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml index e09c01961..88f49cf6b 100644 --- a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml +++ b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster namespace: $namespace stringData: AWS_ACCESS_KEY_ID: minio diff --git a/ramenctl/ramenctl/unconfig.py b/ramenctl/ramenctl/unconfig.py index 82c3714cf..b0abcc671 100644 --- a/ramenctl/ramenctl/unconfig.py +++ b/ramenctl/ramenctl/unconfig.py @@ -24,11 +24,11 @@ def run(args): if env["hub"]: delete_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) - delete_s3_secret([env["hub"]], args) - delete_cloud_credentials(env["clusters"], args) - else: - delete_s3_secret(env["clusters"], args) - delete_cloud_credentials(env["clusters"], args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) + delete_s3_secrets(env["hub"], s3_secrets) + + # TODO: Should be removed by ramen. + delete_cloud_credentials(env["clusters"], args) def delete_hub_dr_resources(hub, clusters, topology): @@ -46,15 +46,21 @@ def delete_hub_dr_resources(hub, clusters, topology): ) -def delete_s3_secret(clusters, args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - yaml = template.substitute(namespace=args.ramen_namespace) - for cluster in clusters: - command.info("Deleting s3 secret in cluster '%s'", cluster) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] + + +def delete_s3_secrets(cluster, secrets): + command.info("Deleting s3 secrets in cluster '%s'", cluster) + for secret in secrets: kubectl.delete( "--filename=-", "--ignore-not-found", - input=yaml, + input=secret, context=cluster, log=command.debug, ) diff --git a/test/addons/recipe/start b/test/addons/recipe/start new file mode 100755 index 000000000..6a8cc0f35 --- /dev/null +++ b/test/addons/recipe/start @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +from drenv import kubectl + +if len(sys.argv) != 2: + sys.exit(f"Usage: {sys.argv[0]} cluster") + +os.chdir(os.path.dirname(__file__)) +cluster = sys.argv[1] + +print("Deploying recipe crd") +kubectl.apply( + "--kustomize", + "https://github.com/RamenDR/recipe.git/config/crd?ref=main&timeout=120s", + context=cluster, +) diff --git a/test/envs/regional-dr-external.yaml.example b/test/envs/regional-dr-external.yaml.example index 1fb9395cc..a928b8ff6 100644 --- a/test/envs/regional-dr-external.yaml.example +++ b/test/envs/regional-dr-external.yaml.example @@ -20,6 +20,7 @@ templates: - addons: - name: ocm-cluster args: ["$name", "hub"] + - name: recipe - addons: - name: cert-manager - name: csi-addons diff --git a/test/envs/regional-dr-hubless.yaml b/test/envs/regional-dr-hubless.yaml index 6bb76bc9c..490ff7b18 100644 --- a/test/envs/regional-dr-hubless.yaml +++ b/test/envs/regional-dr-hubless.yaml @@ -33,6 +33,7 @@ templates: - name: olm - name: minio - name: velero + - name: recipe profiles: - name: "dr1" diff --git a/test/envs/regional-dr-kubevirt.yaml b/test/envs/regional-dr-kubevirt.yaml index 980c3af64..c6527fa3b 100644 --- a/test/envs/regional-dr-kubevirt.yaml +++ b/test/envs/regional-dr-kubevirt.yaml @@ -38,6 +38,7 @@ templates: - name: ocm-cluster args: ["$name", "hub"] - name: cdi + - name: recipe - addons: - name: csi-addons - name: olm diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 0f7e208b3..f42fafe7f 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -31,6 +31,7 @@ templates: - addons: - name: ocm-cluster args: ["$name", "hub"] + - name: recipe - addons: - name: csi-addons - name: olm