Deploy managed clusters s3 secrets using ramen

In OpenShift we deploy 2 s3 secrets (one per s3 store) on the hub, and
the secrets are propagated to the managed clusters using the policy
framework.

In ramenctl we deploy the secrets directly to the managed clusters. This
is much simpler and more reliable, but it bypass the ramen code we want
to test, hiding issues in the real code path.

Change ramenctl to deploy the secrets in the same way as in OpenShift:
- Use 2 secrets, one per cluster s3 store
- Deploy the secrets only on the hub
- Wait until the secrets are propagated to the managed clusters by
  ramen.

With this issues in ramen related code or OCM will break ramenctl early.
Hopefully this will help to detect regressions before they reach QE or
released in OpenShift.

Example run:

    $ ramenctl config $env
    2023-11-22 22:05:19,546 INFO    [ramenctl] Starting config
    2023-11-22 22:05:19,812 INFO    [ramenctl] Waiting until ramen-hub-operator is rolled out
    2023-11-22 22:05:19,889 INFO    [ramenctl] Creating ramen s3 secrets in cluster 'hub'
    2023-11-22 22:05:20,428 INFO    [ramenctl] Updating ramen config map in cluster 'hub'
    2023-11-22 22:05:20,716 INFO    [ramenctl] Creating dr-clusters for regional-dr
    2023-11-22 22:05:20,988 INFO    [ramenctl] Creating dr-policy for regional-dr
    2023-11-22 22:05:21,220 INFO    [ramenctl] Waiting until s3 secrets are propagated to managed clusters
    2023-11-22 22:05:22,800 INFO    [ramenctl] Waiting until DRClusters report phase
    2023-11-22 22:05:22,941 INFO    [ramenctl] Waiting until DRClusters phase is available
    2023-11-22 22:05:23,206 INFO    [ramenctl] Waiting until DRPolicy is validated
    2023-11-22 22:05:23,361 INFO    [ramenctl] Finished config in 3.82 seconds

Signed-off-by: Nir Soffer <nsoffer@redhat.com>

ramenctl/ramenctl/config.py

@@ -21,27 +21,27 @@ def register(commands):
 def run(args):
     env = command.env_info(args)
 
-    s3_secret = generate_ramen_s3_secret(args)
+    s3_secrets = generate_ramen_s3_secrets(env["clusters"], args)
     cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args)
 
     if env["hub"]:
         hub_cm = generate_config_map("hub", env["clusters"], args)
 
         wait_for_ramen_hub_operator(env["hub"], args)
 
-        create_ramen_s3_secret(env["hub"], s3_secret)
-        for cluster in env["clusters"]:
-            create_cloud_credentials_secret(cluster, cloud_secret)
+        create_ramen_s3_secrets(env["hub"], s3_secrets)
+
         create_ramen_config_map(env["hub"], hub_cm)
         create_hub_dr_resources(env["hub"], env["clusters"], env["topology"])
 
+        wait_for_secret_propagation(env["hub"], env["clusters"], args)
         wait_for_dr_clusters(env["hub"], env["clusters"], args)
         wait_for_dr_policy(env["hub"], args)
     else:
         dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args)
 
         for cluster in env["clusters"]:
-            create_ramen_s3_secret(cluster, s3_secret)
+            create_ramen_s3_secrets(cluster, s3_secrets)
             create_cloud_credentials_secret(cluster, cloud_secret)
             create_ramen_config_map(cluster, dr_cluster_cm)
 
@@ -58,14 +58,18 @@ def wait_for_ramen_hub_operator(hub, args):
     )
 
 
-def generate_ramen_s3_secret(args):
+def generate_ramen_s3_secrets(clusters, args):
     template = drenv.template(command.resource("ramen-s3-secret.yaml"))
-    return template.substitute(namespace=args.ramen_namespace)
+    return [
+        template.substitute(namespace=args.ramen_namespace, cluster=cluster)
+        for cluster in clusters
+    ]
 
 
-def create_ramen_s3_secret(cluster, yaml):
-    command.info("Creating ramen s3 secret in cluster '%s'", cluster)
-    kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug)
+def create_ramen_s3_secrets(cluster, secrets):
+    command.info("Creating ramen s3 secrets in cluster '%s'", cluster)
+    for secret in secrets:
+        kubectl.apply("--filename=-", input=secret, context=cluster, log=command.debug)
 
 
 def generate_cloud_credentials_secret(cluster, args):
@@ -111,6 +115,30 @@ def create_hub_dr_resources(hub, clusters, topology):
         kubectl.apply("--filename=-", input=yaml, context=hub, log=command.debug)
 
 
+def wait_for_secret_propagation(hub, clusters, args):
+    command.info("Waiting until s3 secrets are propagated to managed clusters")
+    for cluster in clusters:
+        policy = f"{args.ramen_namespace}.ramen-s3-secret-{cluster}"
+        command.debug("Waiting until policy '%s' reports status", policy)
+        drenv.wait_for(
+            f"policy/{policy}",
+            output="jsonpath={.status}",
+            namespace=cluster,
+            timeout=30,
+            profile=hub,
+            log=command.debug,
+        )
+        command.debug("Waiting until policy %s is compliant", policy)
+        kubectl.wait(
+            f"policy/{policy}",
+            "--for=jsonpath={.status.compliant}=Compliant",
+            "--timeout=30s",
+            f"--namespace={cluster}",
+            context=hub,
+            log=command.debug,
+        )
+
+
 def wait_for_dr_clusters(hub, clusters, args):
     command.info("Waiting until DRClusters report phase")
     for name in clusters:

ramenctl/ramenctl/resources/configmap.yaml

@@ -40,7 +40,7 @@ data:
       s3CompatibleEndpoint: $minio_url_cluster1
       s3Region: us-west-1
       s3SecretRef:
-        name: ramen-s3-secret
+        name: ramen-s3-secret-$cluster1
         namespace: ramen-system
       veleroNamespaceSecretKeyRef:
         key: cloud
@@ -50,7 +50,7 @@ data:
       s3CompatibleEndpoint: $minio_url_cluster2
       s3Region: us-east-1
       s3SecretRef:
-        name: ramen-s3-secret
+        name: ramen-s3-secret-$cluster2
         namespace: ramen-system
       veleroNamespaceSecretKeyRef:
         key: cloud

ramenctl/ramenctl/resources/ramen-s3-secret.yaml

@@ -4,7 +4,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: ramen-s3-secret
+  name: ramen-s3-secret-$cluster
   namespace: $namespace
 stringData:
   AWS_ACCESS_KEY_ID: minio

ramenctl/ramenctl/unconfig.py

@@ -24,11 +24,11 @@ def run(args):
 
     if env["hub"]:
         delete_hub_dr_resources(env["hub"], env["clusters"], env["topology"])
-        delete_s3_secret([env["hub"]], args)
-        delete_cloud_credentials(env["clusters"], args)
-    else:
-        delete_s3_secret(env["clusters"], args)
-        delete_cloud_credentials(env["clusters"], args)
+        s3_secrets = generate_ramen_s3_secrets(env["clusters"], args)
+        delete_s3_secrets(env["hub"], s3_secrets)
+
+    # TODO: Should be removed by ramen.
+    delete_cloud_credentials(env["clusters"], args)
 
 
 def delete_hub_dr_resources(hub, clusters, topology):
@@ -46,15 +46,21 @@ def delete_hub_dr_resources(hub, clusters, topology):
         )
 
 
-def delete_s3_secret(clusters, args):
+def generate_ramen_s3_secrets(clusters, args):
     template = drenv.template(command.resource("ramen-s3-secret.yaml"))
-    yaml = template.substitute(namespace=args.ramen_namespace)
-    for cluster in clusters:
-        command.info("Deleting s3 secret in cluster '%s'", cluster)
+    return [
+        template.substitute(namespace=args.ramen_namespace, cluster=cluster)
+        for cluster in clusters
+    ]
+
+
+def delete_s3_secrets(cluster, secrets):
+    command.info("Deleting s3 secrets in cluster '%s'", cluster)
+    for secret in secrets:
         kubectl.delete(
             "--filename=-",
             "--ignore-not-found",
-            input=yaml,
+            input=secret,
             context=cluster,
             log=command.debug,
         )