From db8beac2e0e05fbb2c5d910d94146c9610f7a459 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sun, 18 Jun 2023 22:49:10 +0300 Subject: [PATCH] Add volsync self test Testing volsync without ramen is the best way to understand how it works and how we need to use it when using minikube (or vanilla k8s?). We use security context to run the busybox app as unprivileged user and set the ownership of the pvc. Without this the busybox app runs as root on minikube. It is possible to allow volsync privileged movers using an annotation on the namespace but this is great pain to use and conflicts with ocm, and there is no reason to run the busybox app as root. The test deploys busybox application on the source cluster, and setup replication to the destination cluster via submariner service export. When the replication is ready we start one replication and wait until it completes. We don't validate the replicated data since the test is complicated and slow as is and it may be too slow for self test. Example run with volsync test environment: $ drenv start envs/volsync.yaml 2023-06-29 23:40:32,314 INFO [volsync] Starting environment 2023-06-29 23:40:32,371 INFO [hub] Starting minikube cluster 2023-06-29 23:40:33,391 INFO [dr1] Starting minikube cluster 2023-06-29 23:40:34,383 INFO [dr2] Starting minikube cluster 2023-06-29 23:41:14,260 INFO [hub] Cluster started in 41.89 seconds 2023-06-29 23:41:14,261 INFO [hub/0] Running addons/submariner/start 2023-06-29 23:43:35,144 INFO [hub/0] addons/submariner/start completed in 140.88 seconds 2023-06-29 23:43:35,145 INFO [hub/0] Running addons/submariner/test 2023-06-29 23:43:55,502 INFO [dr1] Cluster started in 202.11 seconds 2023-06-29 23:44:10,336 INFO [dr2] Cluster started in 215.95 seconds 2023-06-29 23:44:16,476 INFO [hub/0] addons/submariner/test completed in 41.33 seconds 2023-06-29 23:44:16,477 INFO [volsync/0] Running addons/volsync/start 2023-06-29 23:44:48,775 INFO [volsync/0] addons/volsync/start completed in 32.30 seconds 2023-06-29 23:44:48,775 INFO [volsync/0] Running addons/volsync/test 2023-06-29 23:46:08,017 INFO [volsync/0] addons/volsync/test completed in 79.24 seconds 2023-06-29 23:46:08,018 INFO [volsync] Environment started in 335.70 seconds Example run with regional-dr environment: 2023-06-30 00:06:13,967 INFO [rdr/0] Running addons/rbd-mirror/start 2023-06-30 00:06:13,968 INFO [rdr/1] Running addons/volsync/start 2023-06-30 00:06:48,121 INFO [rdr/1] addons/volsync/start completed in 34.15 seconds 2023-06-30 00:06:48,121 INFO [rdr/1] Running addons/volsync/test 2023-06-30 00:07:11,942 INFO [rdr/0] addons/rbd-mirror/start completed in 57.98 seconds 2023-06-30 00:07:11,942 INFO [rdr/0] Running addons/rbd-mirror/test 2023-06-30 00:07:24,297 INFO [rdr/1] addons/volsync/test completed in 36.18 seconds 2023-06-30 00:07:28,676 INFO [rdr/0] addons/rbd-mirror/test completed in 16.73 seconds Signed-off-by: Nir Soffer --- .../volsync/destination/kustomization.yaml | 7 + .../addons/volsync/destination/namespace.yaml | 8 + .../volsync/destination/replication-dst.yaml | 20 +++ test/addons/volsync/source/deploy.yaml | 47 +++++ test/addons/volsync/source/kustomization.yaml | 8 + test/addons/volsync/source/namespace.yaml | 8 + test/addons/volsync/source/pvc.yaml | 17 ++ .../volsync/source/replication-src.yaml | 22 +++ test/addons/volsync/source/secret.yaml | 12 ++ test/addons/volsync/test | 170 ++++++++++++++++++ 10 files changed, 319 insertions(+) create mode 100644 test/addons/volsync/destination/kustomization.yaml create mode 100644 test/addons/volsync/destination/namespace.yaml create mode 100644 test/addons/volsync/destination/replication-dst.yaml create mode 100644 test/addons/volsync/source/deploy.yaml create mode 100644 test/addons/volsync/source/kustomization.yaml create mode 100644 test/addons/volsync/source/namespace.yaml create mode 100644 test/addons/volsync/source/pvc.yaml create mode 100644 test/addons/volsync/source/replication-src.yaml create mode 100644 test/addons/volsync/source/secret.yaml create mode 100755 test/addons/volsync/test diff --git a/test/addons/volsync/destination/kustomization.yaml b/test/addons/volsync/destination/kustomization.yaml new file mode 100644 index 0000000000..15d4377777 --- /dev/null +++ b/test/addons/volsync/destination/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: +- namespace.yaml +- replication-dst.yaml diff --git a/test/addons/volsync/destination/namespace.yaml b/test/addons/volsync/destination/namespace.yaml new file mode 100644 index 0000000000..b3403dc81f --- /dev/null +++ b/test/addons/volsync/destination/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: busybox diff --git a/test/addons/volsync/destination/replication-dst.yaml b/test/addons/volsync/destination/replication-dst.yaml new file mode 100644 index 0000000000..ec418efe00 --- /dev/null +++ b/test/addons/volsync/destination/replication-dst.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationDestination +metadata: + name: busybox-dst + namespace: busybox +spec: + rsyncTLS: + copyMethod: Snapshot + capacity: 1Gi + accessModes: [ReadWriteOnce] + storageClassName: csi-hostpath-sc + volumeSnapshotClassName: csi-hostpath-snapclass + moverSecurityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/deploy.yaml b/test/addons/volsync/source/deploy.yaml new file mode 100644 index 0000000000..7820a30abc --- /dev/null +++ b/test/addons/volsync/source/deploy.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + appname: busybox + name: busybox + namespace: busybox +spec: + replicas: 1 + selector: + matchLabels: + appname: busybox + template: + metadata: + labels: + appname: busybox + spec: + containers: + - image: docker.io/library/busybox:latest + imagePullPolicy: IfNotPresent + name: busybox + command: + - sh + - -c + - | + trap exit TERM + while true; do + echo $(date) | tee -a /mnt/test/outfile + sync + sleep 10 & + wait + done + volumeMounts: + - name: volume + mountPath: /mnt/test + volumes: + - name: volume + persistentVolumeClaim: + claimName: busybox-pvc + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/kustomization.yaml b/test/addons/volsync/source/kustomization.yaml new file mode 100644 index 0000000000..bce64ee0ac --- /dev/null +++ b/test/addons/volsync/source/kustomization.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: +- namespace.yaml +- pvc.yaml +- deploy.yaml diff --git a/test/addons/volsync/source/namespace.yaml b/test/addons/volsync/source/namespace.yaml new file mode 100644 index 0000000000..b3403dc81f --- /dev/null +++ b/test/addons/volsync/source/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: busybox diff --git a/test/addons/volsync/source/pvc.yaml b/test/addons/volsync/source/pvc.yaml new file mode 100644 index 0000000000..753849b27d --- /dev/null +++ b/test/addons/volsync/source/pvc.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: busybox-pvc + namespace: busybox + labels: + appname: busybox +spec: + accessModes: [ReadWriteOnce] + storageClassName: csi-hostpath-sc + resources: + requests: + storage: 1Gi diff --git a/test/addons/volsync/source/replication-src.yaml b/test/addons/volsync/source/replication-src.yaml new file mode 100644 index 0000000000..7d5596a162 --- /dev/null +++ b/test/addons/volsync/source/replication-src.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: busybox-src + namespace: busybox +spec: + sourcePVC: busybox-pvc + trigger: + manual: replication-1 + rsyncTLS: + keySecret: volsync-rsync-tls-busybox-dst + address: volsync-rsync-tls-dst-busybox-dst.busybox.svc.clusterset.local + copyMethod: Snapshot + volumeSnapshotClassName: csi-hostpath-snapclass + moverSecurityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/secret.yaml b/test/addons/volsync/source/secret.yaml new file mode 100644 index 0000000000..11213866d0 --- /dev/null +++ b/test/addons/volsync/source/secret.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: volsync-rsync-tls-busybox-dst + namespace: busybox +data: + psk.txt: $value diff --git a/test/addons/volsync/test b/test/addons/volsync/test new file mode 100755 index 0000000000..50e8f42281 --- /dev/null +++ b/test/addons/volsync/test @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import sys + +import drenv +from drenv import kubectl +from drenv import subctl + +DEPLOY = "busybox" +NAMESPACE = "busybox" + +VOLSYNC_SECRET = "volsync-rsync-tls-busybox-dst" +VOLSYNC_SERVICE = "volsync-rsync-tls-dst-busybox-dst" + + +def setup_application(cluster): + print(f"Deploy application on cluster '{cluster}'") + kubectl.apply("--kustomize", "source", context=cluster) + + +def setup_replication_destination(cluster): + print(f"Create replication destination on cluster '{cluster}'") + kubectl.apply("--kustomize", "destination", context=cluster) + + +def wait_for_application(cluster): + print(f"Waiting until deploy '{DEPLOY}' is rolled out in cluster '{cluster}'") + kubectl.rollout( + "status", + f"deploy/{DEPLOY}", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + + +def wait_for_replication_destination(cluster): + print( + f"Waiting until replication destination is synchronizing in cluster '{cluster}'" + ) + kubectl.wait( + "replicationdestination/busybox-dst", + "--for=condition=Synchronizing=True", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + + +def setup_replication_secret(cluster1, cluster2): + """ + Create a secret in the source cluster using data from the secret created by + volsync on the destiantion cluster. + """ + print(f"Getting volsync secret in cluster '{cluster2}'") + psk_txt = kubectl.get( + f"secret/{VOLSYNC_SECRET}", + f"--namespace={NAMESPACE}", + "--output=jsonpath={.data.psk\\.txt}", + context=cluster2, + ) + + print(f"Creating volsync secret in cluster '{cluster1}'") + template = drenv.template("source/secret.yaml") + yaml = template.substitute(value=psk_txt) + kubectl.apply("--filename=-", input=yaml, context=cluster1) + + +def setup_replication_service(cluster1, cluster2): + """ + Export volsync replication service from the destination cluster to the + source cluster using submariner. + """ + print(f"Exporting volsync service in cluster '{cluster2}'") + subctl.export("service", VOLSYNC_SERVICE, cluster2, namespace=NAMESPACE) + + print(f"Waiting until service export is synced in cluster '{cluster2}'") + kubectl.wait( + f"serviceexports/{VOLSYNC_SERVICE}", + "--for=condition=Synced=True", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster2, + ) + + print(f"Waiting until service import is ready in cluster '{cluster1}'") + drenv.wait_for( + f"serviceimports/{VOLSYNC_SERVICE}", + output="jsonpath={.status.clusters}", + namespace=NAMESPACE, + timeout=120, + profile=cluster1, + ) + + +def run_replication(cluster): + """ + Start replication and wait until replication completes. + """ + print(f"Creating replication source in cluster '{cluster}'") + kubectl.apply("--filename", "source/replication-src.yaml", context=cluster) + + print(f"Waiting until replication is completed in cluster '{cluster}'") + kubectl.wait( + "replicationsource/busybox-src", + "--for=jsonpath={.status.lastManualSync}=replication-1", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + out = kubectl.get( + "replicationsource/busybox-src", + "--output=jsonpath={.status}", + f"--namespace={NAMESPACE}", + context=cluster, + ) + status = json.loads(out) + print("Replication status:") + print(json.dumps(status, indent=2)) + + +def teardown(cluster1, cluster2): + """ + Remove deployments from both clusters. This also deletes additonal + resources created in the same namespace. + """ + print("Cleaning up clusters") + subctl.unexport("service", VOLSYNC_SERVICE, cluster2, namespace=NAMESPACE) + kubectl.delete( + "--kustomize", + "source", + "--ignore-not-found", + "--wait=false", + context=cluster1, + ) + kubectl.delete( + "--kustomize", + "destination", + "--ignore-not-found", + "--wait=false", + context=cluster2, + ) + kubectl.delete("--kustomize", "source", "--ignore-not-found", context=cluster1) + kubectl.delete("--kustomize", "destination", "--ignore-not-found", context=cluster2) + + +if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} cluster1 cluster2") + sys.exit(1) + +os.chdir(os.path.dirname(__file__)) +cluster1, cluster2 = sys.argv[1:] + +setup_application(cluster1) +setup_replication_destination(cluster2) + +wait_for_application(cluster1) +wait_for_replication_destination(cluster2) + +setup_replication_secret(cluster1, cluster2) +setup_replication_service(cluster1, cluster2) + +run_replication(cluster1) + +teardown(cluster1, cluster2)