From d0de64f0cf3a07599d36f9512d8a7c3316f27d60 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 22 Nov 2023 21:38:53 +0200 Subject: [PATCH] Deploy managed clusters s3 secrets using ramen In OpenShift we deploy 2 s3 secrets (one per s3 store) on the hub, and the secrets are propagated to the managed clusters using the policy framework. In ramenctl we deploy the secrets directly to the managed clusters. This is much simpler and more reliable, but it bypass the ramen code we want to test, hiding issues in the real code path. Change ramenctl to deploy the secrets in the same way as in OpenShift: - Use 2 secrets, one per cluster s3 store - Deploy the secrets only on the hub - Wait until the secrets are propagated to the managed clusters by ramen. With this issues in ramen related code or OCM will break ramenctl early. Hopefully this will help to detect regressions before they reach QE or released in OpenShift. Example run: $ ramenctl config $env 2023-11-22 22:05:19,546 INFO [ramenctl] Starting config 2023-11-22 22:05:19,812 INFO [ramenctl] Waiting until ramen-hub-operator is rolled out 2023-11-22 22:05:19,889 INFO [ramenctl] Creating ramen s3 secrets in cluster 'hub' 2023-11-22 22:05:20,428 INFO [ramenctl] Updating ramen config map in cluster 'hub' 2023-11-22 22:05:20,716 INFO [ramenctl] Creating dr-clusters for regional-dr 2023-11-22 22:05:20,988 INFO [ramenctl] Creating dr-policy for regional-dr 2023-11-22 22:05:21,220 INFO [ramenctl] Waiting until s3 secrets are propagated to managed clusters 2023-11-22 22:05:22,800 INFO [ramenctl] Waiting until DRClusters report phase 2023-11-22 22:05:22,941 INFO [ramenctl] Waiting until DRClusters phase is available 2023-11-22 22:05:23,206 INFO [ramenctl] Waiting until DRPolicy is validated 2023-11-22 22:05:23,361 INFO [ramenctl] Finished config in 3.82 seconds Signed-off-by: Nir Soffer --- ramenctl/ramenctl/config.py | 48 +++++++++++++++---- ramenctl/ramenctl/resources/configmap.yaml | 4 +- .../ramenctl/resources/ramen-s3-secret.yaml | 2 +- ramenctl/ramenctl/unconfig.py | 26 ++++++---- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/ramenctl/ramenctl/config.py b/ramenctl/ramenctl/config.py index 7c5ef4fff..f3b0b2f91 100644 --- a/ramenctl/ramenctl/config.py +++ b/ramenctl/ramenctl/config.py @@ -21,7 +21,7 @@ def register(commands): def run(args): env = command.env_info(args) - s3_secret = generate_ramen_s3_secret(args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) cloud_secret = generate_cloud_credentials_secret(env["clusters"][0], args) if env["hub"]: @@ -29,19 +29,19 @@ def run(args): wait_for_ramen_hub_operator(env["hub"], args) - create_ramen_s3_secret(env["hub"], s3_secret) - for cluster in env["clusters"]: - create_cloud_credentials_secret(cluster, cloud_secret) + create_ramen_s3_secrets(env["hub"], s3_secrets) + create_ramen_config_map(env["hub"], hub_cm) create_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) + wait_for_secret_propagation(env["hub"], env["clusters"], args) wait_for_dr_clusters(env["hub"], env["clusters"], args) wait_for_dr_policy(env["hub"], args) else: dr_cluster_cm = generate_config_map("dr-cluster", env["clusters"], args) for cluster in env["clusters"]: - create_ramen_s3_secret(cluster, s3_secret) + create_ramen_s3_secrets(cluster, s3_secrets) create_cloud_credentials_secret(cluster, cloud_secret) create_ramen_config_map(cluster, dr_cluster_cm) @@ -58,14 +58,18 @@ def wait_for_ramen_hub_operator(hub, args): ) -def generate_ramen_s3_secret(args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - return template.substitute(namespace=args.ramen_namespace) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] -def create_ramen_s3_secret(cluster, yaml): - command.info("Creating ramen s3 secret in cluster '%s'", cluster) - kubectl.apply("--filename=-", input=yaml, context=cluster, log=command.debug) +def create_ramen_s3_secrets(cluster, secrets): + command.info("Creating ramen s3 secrets in cluster '%s'", cluster) + for secret in secrets: + kubectl.apply("--filename=-", input=secret, context=cluster, log=command.debug) def generate_cloud_credentials_secret(cluster, args): @@ -111,6 +115,30 @@ def create_hub_dr_resources(hub, clusters, topology): kubectl.apply("--filename=-", input=yaml, context=hub, log=command.debug) +def wait_for_secret_propagation(hub, clusters, args): + command.info("Waiting until s3 secrets are propagated to managed clusters") + for cluster in clusters: + policy = f"{args.ramen_namespace}.ramen-s3-secret-{cluster}" + command.debug("Waiting until policy '%s' reports status", policy) + drenv.wait_for( + f"policy/{policy}", + output="jsonpath={.status}", + namespace=cluster, + timeout=30, + profile=hub, + log=command.debug, + ) + command.debug("Waiting until policy %s is compliant", policy) + kubectl.wait( + f"policy/{policy}", + "--for=jsonpath={.status.compliant}=Compliant", + "--timeout=30s", + f"--namespace={cluster}", + context=hub, + log=command.debug, + ) + + def wait_for_dr_clusters(hub, clusters, args): command.info("Waiting until DRClusters report phase") for name in clusters: diff --git a/ramenctl/ramenctl/resources/configmap.yaml b/ramenctl/ramenctl/resources/configmap.yaml index d10b180ee..c82136f1f 100644 --- a/ramenctl/ramenctl/resources/configmap.yaml +++ b/ramenctl/ramenctl/resources/configmap.yaml @@ -40,7 +40,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster1 s3Region: us-west-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster1 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud @@ -50,7 +50,7 @@ data: s3CompatibleEndpoint: $minio_url_cluster2 s3Region: us-east-1 s3SecretRef: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster2 namespace: ramen-system veleroNamespaceSecretKeyRef: key: cloud diff --git a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml index e09c01961..88f49cf6b 100644 --- a/ramenctl/ramenctl/resources/ramen-s3-secret.yaml +++ b/ramenctl/ramenctl/resources/ramen-s3-secret.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: ramen-s3-secret + name: ramen-s3-secret-$cluster namespace: $namespace stringData: AWS_ACCESS_KEY_ID: minio diff --git a/ramenctl/ramenctl/unconfig.py b/ramenctl/ramenctl/unconfig.py index 82c3714cf..b0abcc671 100644 --- a/ramenctl/ramenctl/unconfig.py +++ b/ramenctl/ramenctl/unconfig.py @@ -24,11 +24,11 @@ def run(args): if env["hub"]: delete_hub_dr_resources(env["hub"], env["clusters"], env["topology"]) - delete_s3_secret([env["hub"]], args) - delete_cloud_credentials(env["clusters"], args) - else: - delete_s3_secret(env["clusters"], args) - delete_cloud_credentials(env["clusters"], args) + s3_secrets = generate_ramen_s3_secrets(env["clusters"], args) + delete_s3_secrets(env["hub"], s3_secrets) + + # TODO: Should be removed by ramen. + delete_cloud_credentials(env["clusters"], args) def delete_hub_dr_resources(hub, clusters, topology): @@ -46,15 +46,21 @@ def delete_hub_dr_resources(hub, clusters, topology): ) -def delete_s3_secret(clusters, args): +def generate_ramen_s3_secrets(clusters, args): template = drenv.template(command.resource("ramen-s3-secret.yaml")) - yaml = template.substitute(namespace=args.ramen_namespace) - for cluster in clusters: - command.info("Deleting s3 secret in cluster '%s'", cluster) + return [ + template.substitute(namespace=args.ramen_namespace, cluster=cluster) + for cluster in clusters + ] + + +def delete_s3_secrets(cluster, secrets): + command.info("Deleting s3 secrets in cluster '%s'", cluster) + for secret in secrets: kubectl.delete( "--filename=-", "--ignore-not-found", - input=yaml, + input=secret, context=cluster, log=command.debug, )