Deploy managed clusters s3 secrets using ramen

ramenctl/ramenctl/config.py

@@ -21,27 +21,27 @@ def register(commands):
 def run(args):
     env = command.env_info(args)
 
-    s3_secret = generate_ramen_s3_secret(args)
+    s3_secrets = generate_ramen_s3_secrets(env["clusters"], args)
     cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args)
 
     if env["hub"]:
         hub_cm = generate_config_map("hub", env["clusters"], args)
 
         wait_for_ramen_hub_operator(env["hub"], args)
 
-        create_ramen_s3_secret(env["hub"], s3_secret)
-        for cluster in env["clusters"]:
-            create_cloud_credentials_secret(cluster, cloud_secret)
+        create_ramen_s3_secrets(env["hub"], s3_secrets)
+
         create_ramen_config_map(env["hub"], hub_cm)
         create_hub_dr_resources(env["hub"], env["clusters"], env["topology"])
 
+        wait_for_secret_propagation(env["hub"], env["clusters"], args)
         wait_for_dr_clusters(env["hub"], env["clusters"], args)
         wait_for_dr_policy(env["hub"], args)
     else:
         dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args)
 
         for cluster in env["clusters"]:
-            create_ramen_s3_secret(cluster, s3_secret)
+            create_ramen_s3_secrets(cluster, s3_secrets)
             create_cloud_credentials_secret(cluster, cloud_secret)
             create_ramen_config_map(cluster, dr_cluster_cm)
 
@@ -58,14 +58,18 @@ def wait_for_ramen_hub_operator(hub, args):
     )
 
 
-def generate_ramen_s3_secret(args):
+def generate_ramen_s3_secrets(clusters, args):
     template = drenv.template(command.resource("ramen-s3-secret.yaml"))
-    return template.substitute(namespace=args.ramen_namespace)
+    return [
+        template.substitute(namespace=args.ramen_namespace, cluster=cluster)
+        for cluster in clusters
+    ]
 
 
-def create_ramen_s3_secret(cluster, yaml):
-    command.info("Creating ramen s3 secret in cluster '%s'", cluster)
-    kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug)
+def create_ramen_s3_secrets(cluster, secrets):
+    command.info("Creating ramen s3 secrets in cluster '%s'", cluster)
+    for secret in secrets:
+        kubectl.apply("--filename=-", input=secret, context=cluster, log=command.debug)
 
 
 def generate_cloud_credentials_secret(cluster, args):
@@ -111,6 +115,30 @@ def create_hub_dr_resources(hub, clusters, topology):
         kubectl.apply("--filename=-", input=yaml, context=hub, log=command.debug)
 
 
+def wait_for_secret_propagation(hub, clusters, args):
+    command.info("Waiting until s3 secrets are propagated to managed clusters")
+    for cluster in clusters:
+        policy = f"{args.ramen_namespace}.ramen-s3-secret-{cluster}"
+        command.debug("Waiting until policy '%s' reports status", policy)
+        drenv.wait_for(
+            f"policy/{policy}",
+            output="jsonpath={.status}",
+            namespace=cluster,
+            timeout=30,
+            profile=hub,
+            log=command.debug,
+        )
+        command.debug("Waiting until policy %s is compliant", policy)
+        kubectl.wait(
+            f"policy/{policy}",
+            "--for=jsonpath={.status.compliant}=Compliant",
+            "--timeout=30s",
+            f"--namespace={cluster}",
+            context=hub,
+            log=command.debug,
+        )
+
+
 def wait_for_dr_clusters(hub, clusters, args):
     command.info("Waiting until DRClusters report phase")
     for name in clusters:

ramenctl/ramenctl/resources/configmap.yaml

@@ -40,7 +40,7 @@ data:
       s3CompatibleEndpoint: $minio_url_cluster1
       s3Region: us-west-1
       s3SecretRef:
-        name: ramen-s3-secret
+        name: ramen-s3-secret-$cluster1
         namespace: ramen-system
       veleroNamespaceSecretKeyRef:
         key: cloud
@@ -50,7 +50,7 @@ data:
       s3CompatibleEndpoint: $minio_url_cluster2
       s3Region: us-east-1
       s3SecretRef:
-        name: ramen-s3-secret
+        name: ramen-s3-secret-$cluster2
         namespace: ramen-system
       veleroNamespaceSecretKeyRef:
         key: cloud

ramenctl/ramenctl/resources/ramen-s3-secret.yaml

@@ -4,7 +4,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: ramen-s3-secret
+  name: ramen-s3-secret-$cluster
   namespace: $namespace
 stringData:
   AWS_ACCESS_KEY_ID: minio

ramenctl/ramenctl/unconfig.py

@@ -24,11 +24,11 @@ def run(args):
 
     if env["hub"]:
         delete_hub_dr_resources(env["hub"], env["clusters"], env["topology"])
-        delete_s3_secret([env["hub"]], args)
-        delete_cloud_credentials(env["clusters"], args)
-    else:
-        delete_s3_secret(env["clusters"], args)
-        delete_cloud_credentials(env["clusters"], args)
+        s3_secrets = generate_ramen_s3_secrets(env["clusters"], args)
+        delete_s3_secrets(env["hub"], s3_secrets)
+
+    # TODO: Should be removed by ramen.
+    delete_cloud_credentials(env["clusters"], args)
 
 
 def delete_hub_dr_resources(hub, clusters, topology):
@@ -46,15 +46,21 @@ def delete_hub_dr_resources(hub, clusters, topology):
         )
 
 
-def delete_s3_secret(clusters, args):
+def generate_ramen_s3_secrets(clusters, args):
     template = drenv.template(command.resource("ramen-s3-secret.yaml"))
-    yaml = template.substitute(namespace=args.ramen_namespace)
-    for cluster in clusters:
-        command.info("Deleting s3 secret in cluster '%s'", cluster)
+    return [
+        template.substitute(namespace=args.ramen_namespace, cluster=cluster)
+        for cluster in clusters
+    ]
+
+
+def delete_s3_secrets(cluster, secrets):
+    command.info("Deleting s3 secrets in cluster '%s'", cluster)
+    for secret in secrets:
         kubectl.delete(
             "--filename=-",
             "--ignore-not-found",
-            input=yaml,
+            input=secret,
             context=cluster,
             log=command.debug,
         )

test/addons/recipe/start

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+
+from drenv import kubectl
+
+if len(sys.argv) != 2:
+    sys.exit(f"Usage: {sys.argv[0]} cluster")
+
+os.chdir(os.path.dirname(__file__))
+cluster = sys.argv[1]
+
+print("Deploying recipe crd")
+kubectl.apply(
+    "--kustomize",
+    "https://github.com/RamenDR/recipe.git/config/crd?ref=main&timeout=120s",
+    context=cluster,
+)

test/envs/regional-dr-external.yaml.example

@@ -20,6 +20,7 @@ templates:
       - addons:
           - name: ocm-cluster
             args: ["$name", "hub"]
+          - name: recipe
       - addons:
           - name: cert-manager
           - name: csi-addons

test/envs/regional-dr-hubless.yaml

@@ -33,6 +33,7 @@ templates:
           - name: olm
           - name: minio
           - name: velero
+          - name: recipe
 
 profiles:
   - name: "dr1"

test/envs/regional-dr-kubevirt.yaml

@@ -38,6 +38,7 @@ templates:
           - name: ocm-cluster
             args: ["$name", "hub"]
           - name: cdi
+          - name: recipe
       - addons:
           - name: csi-addons
           - name: olm

test/envs/regional-dr.yaml

@@ -31,6 +31,7 @@ templates:
       - addons:
           - name: ocm-cluster
             args: ["$name", "hub"]
+          - name: recipe
       - addons:
           - name: csi-addons
           - name: olm