diff --git a/test/README.md b/test/README.md index 4c4852f0a..861b07284 100644 --- a/test/README.md +++ b/test/README.md @@ -45,6 +45,14 @@ environment. [Velero Basic Install](https://velero.io/docs/v1.11/basic-install/) for the details. Version 1.9.3 or later is required. +1. Install `helm` tool - on Fedora you can use: + + ``` + dnf install helm + ``` + + See [Installing Helm](https://helm.sh/docs/intro/install/) for other options. + 1. Install `docker` ``` @@ -574,6 +582,7 @@ simpler and faster to work with a minimal environment. - `rook.yaml` - for testing `rook` deployment - `submariner.yaml` - for testing `submariner` deployment - `velero.yaml` - for testing `velero` deployment +- `volsync.yaml` - for testing `volsync` deployment ## Testing drenv diff --git a/test/addons/submariner/test b/test/addons/submariner/test index 1d4add945..0e813c200 100755 --- a/test/addons/submariner/test +++ b/test/addons/submariner/test @@ -98,7 +98,7 @@ def wait_for_service(cluster, namespace): f"deploy/{SERVICE}", "--for=condition=Available", f"--namespace={namespace}", - "--timeout=60s", + "--timeout=120s", context=cluster, ) @@ -109,7 +109,7 @@ def wait_for_pod(cluster, namespace): "pod/test", "--for=condition=Ready", f"--namespace={namespace}", - "--timeout=60s", + "--timeout=120s", context=cluster, ) @@ -130,7 +130,7 @@ def wait_for_service_export(cluster, namespace): f"serviceexports/{SERVICE}", "--for=condition=Synced=True", f"--namespace={namespace}", - "--timeout=60s", + "--timeout=120s", context=cluster, ) exports = kubectl.describe( @@ -147,7 +147,7 @@ def wait_for_service_import(cluster, namespace): f"serviceimports/{SERVICE}", output="jsonpath={.status.clusters}", namespace=namespace, - timeout=60, + timeout=120, profile=cluster, ) imports = kubectl.describe( diff --git a/test/addons/volsync/destination/kustomization.yaml b/test/addons/volsync/destination/kustomization.yaml new file mode 100644 index 000000000..15d437777 --- /dev/null +++ b/test/addons/volsync/destination/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: +- namespace.yaml +- replication-dst.yaml diff --git a/test/addons/volsync/destination/namespace.yaml b/test/addons/volsync/destination/namespace.yaml new file mode 100644 index 000000000..b3403dc81 --- /dev/null +++ b/test/addons/volsync/destination/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: busybox diff --git a/test/addons/volsync/destination/replication-dst.yaml b/test/addons/volsync/destination/replication-dst.yaml new file mode 100644 index 000000000..ec418efe0 --- /dev/null +++ b/test/addons/volsync/destination/replication-dst.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationDestination +metadata: + name: busybox-dst + namespace: busybox +spec: + rsyncTLS: + copyMethod: Snapshot + capacity: 1Gi + accessModes: [ReadWriteOnce] + storageClassName: csi-hostpath-sc + volumeSnapshotClassName: csi-hostpath-snapclass + moverSecurityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/deploy.yaml b/test/addons/volsync/source/deploy.yaml new file mode 100644 index 000000000..7820a30ab --- /dev/null +++ b/test/addons/volsync/source/deploy.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + appname: busybox + name: busybox + namespace: busybox +spec: + replicas: 1 + selector: + matchLabels: + appname: busybox + template: + metadata: + labels: + appname: busybox + spec: + containers: + - image: docker.io/library/busybox:latest + imagePullPolicy: IfNotPresent + name: busybox + command: + - sh + - -c + - | + trap exit TERM + while true; do + echo $(date) | tee -a /mnt/test/outfile + sync + sleep 10 & + wait + done + volumeMounts: + - name: volume + mountPath: /mnt/test + volumes: + - name: volume + persistentVolumeClaim: + claimName: busybox-pvc + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/kustomization.yaml b/test/addons/volsync/source/kustomization.yaml new file mode 100644 index 000000000..bce64ee0a --- /dev/null +++ b/test/addons/volsync/source/kustomization.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: +- namespace.yaml +- pvc.yaml +- deploy.yaml diff --git a/test/addons/volsync/source/namespace.yaml b/test/addons/volsync/source/namespace.yaml new file mode 100644 index 000000000..b3403dc81 --- /dev/null +++ b/test/addons/volsync/source/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: busybox diff --git a/test/addons/volsync/source/pvc.yaml b/test/addons/volsync/source/pvc.yaml new file mode 100644 index 000000000..753849b27 --- /dev/null +++ b/test/addons/volsync/source/pvc.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: busybox-pvc + namespace: busybox + labels: + appname: busybox +spec: + accessModes: [ReadWriteOnce] + storageClassName: csi-hostpath-sc + resources: + requests: + storage: 1Gi diff --git a/test/addons/volsync/source/replication-src.yaml b/test/addons/volsync/source/replication-src.yaml new file mode 100644 index 000000000..7d5596a16 --- /dev/null +++ b/test/addons/volsync/source/replication-src.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: busybox-src + namespace: busybox +spec: + sourcePVC: busybox-pvc + trigger: + manual: replication-1 + rsyncTLS: + keySecret: volsync-rsync-tls-busybox-dst + address: volsync-rsync-tls-dst-busybox-dst.busybox.svc.clusterset.local + copyMethod: Snapshot + volumeSnapshotClassName: csi-hostpath-snapclass + moverSecurityContext: + runAsUser: 10000 + runAsGroup: 10000 + fsGroup: 10000 diff --git a/test/addons/volsync/source/secret.yaml b/test/addons/volsync/source/secret.yaml new file mode 100644 index 000000000..11213866d --- /dev/null +++ b/test/addons/volsync/source/secret.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: volsync-rsync-tls-busybox-dst + namespace: busybox +data: + psk.txt: $value diff --git a/test/addons/volsync/start b/test/addons/volsync/start index ca8a682cf..d0a1899ed 100755 --- a/test/addons/volsync/start +++ b/test/addons/volsync/start @@ -6,36 +6,66 @@ import os import sys +from drenv import commands from drenv import kubectl -CRDS = [ - "../../../hack/test/volsync.backube_replicationsources.yaml", - "../../../hack/test/volsync.backube_replicationdestinations.yaml", -] +NAMESPACE = "volsync-system" +DEPLOYMENT = "volsync" -def deploy(cluster): - print("Deploying volsync crds") - for crd in CRDS: - kubectl.apply(f"--filename={crd}", context=cluster) +def add_helm_repo(): + print("Adding helm backube repo") + cmd = [ + "helm", + "repo", + "add", + "backube", + "https://backube.github.io/helm-charts/", + ] + for line in commands.watch(*cmd): + print(line) -def wait(cluster): - print("Waiting until cdrs are established") - for crd in CRDS: - kubectl.wait( - "--for=condition=established", - f"--filename={crd}", - context=cluster, - ) +def install_volsync(cluster): + print(f"Installing volsync in cluster '{cluster}'") + cmd = [ + "helm", + "upgrade", + "--install", + "--create-namespace", + "--namespace", + NAMESPACE, + DEPLOYMENT, + "backube/volsync", + "--kube-context", + cluster, + ] + for line in commands.watch(*cmd): + print(line) -if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} cluster") +def wait_for_deployment(cluster): + print(f"Waiting until deployment {DEPLOYMENT} is rolled out in cluster '{cluster}'") + kubectl.rollout( + "status", + f"deploy/{DEPLOYMENT}", + f"--namespace={NAMESPACE}", + "--timeout=300s", + context=cluster, + ) + + +if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} cluster1 cluster2") sys.exit(1) os.chdir(os.path.dirname(__file__)) -cluster = sys.argv[1] +clusters = sys.argv[1:] + +add_helm_repo() + +for cluster in clusters: + install_volsync(cluster) -deploy(cluster) -wait(cluster) +for cluster in clusters: + wait_for_deployment(cluster) diff --git a/test/addons/volsync/test b/test/addons/volsync/test new file mode 100755 index 000000000..50e8f4228 --- /dev/null +++ b/test/addons/volsync/test @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import sys + +import drenv +from drenv import kubectl +from drenv import subctl + +DEPLOY = "busybox" +NAMESPACE = "busybox" + +VOLSYNC_SECRET = "volsync-rsync-tls-busybox-dst" +VOLSYNC_SERVICE = "volsync-rsync-tls-dst-busybox-dst" + + +def setup_application(cluster): + print(f"Deploy application on cluster '{cluster}'") + kubectl.apply("--kustomize", "source", context=cluster) + + +def setup_replication_destination(cluster): + print(f"Create replication destination on cluster '{cluster}'") + kubectl.apply("--kustomize", "destination", context=cluster) + + +def wait_for_application(cluster): + print(f"Waiting until deploy '{DEPLOY}' is rolled out in cluster '{cluster}'") + kubectl.rollout( + "status", + f"deploy/{DEPLOY}", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + + +def wait_for_replication_destination(cluster): + print( + f"Waiting until replication destination is synchronizing in cluster '{cluster}'" + ) + kubectl.wait( + "replicationdestination/busybox-dst", + "--for=condition=Synchronizing=True", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + + +def setup_replication_secret(cluster1, cluster2): + """ + Create a secret in the source cluster using data from the secret created by + volsync on the destiantion cluster. + """ + print(f"Getting volsync secret in cluster '{cluster2}'") + psk_txt = kubectl.get( + f"secret/{VOLSYNC_SECRET}", + f"--namespace={NAMESPACE}", + "--output=jsonpath={.data.psk\\.txt}", + context=cluster2, + ) + + print(f"Creating volsync secret in cluster '{cluster1}'") + template = drenv.template("source/secret.yaml") + yaml = template.substitute(value=psk_txt) + kubectl.apply("--filename=-", input=yaml, context=cluster1) + + +def setup_replication_service(cluster1, cluster2): + """ + Export volsync replication service from the destination cluster to the + source cluster using submariner. + """ + print(f"Exporting volsync service in cluster '{cluster2}'") + subctl.export("service", VOLSYNC_SERVICE, cluster2, namespace=NAMESPACE) + + print(f"Waiting until service export is synced in cluster '{cluster2}'") + kubectl.wait( + f"serviceexports/{VOLSYNC_SERVICE}", + "--for=condition=Synced=True", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster2, + ) + + print(f"Waiting until service import is ready in cluster '{cluster1}'") + drenv.wait_for( + f"serviceimports/{VOLSYNC_SERVICE}", + output="jsonpath={.status.clusters}", + namespace=NAMESPACE, + timeout=120, + profile=cluster1, + ) + + +def run_replication(cluster): + """ + Start replication and wait until replication completes. + """ + print(f"Creating replication source in cluster '{cluster}'") + kubectl.apply("--filename", "source/replication-src.yaml", context=cluster) + + print(f"Waiting until replication is completed in cluster '{cluster}'") + kubectl.wait( + "replicationsource/busybox-src", + "--for=jsonpath={.status.lastManualSync}=replication-1", + f"--namespace={NAMESPACE}", + "--timeout=120s", + context=cluster, + ) + out = kubectl.get( + "replicationsource/busybox-src", + "--output=jsonpath={.status}", + f"--namespace={NAMESPACE}", + context=cluster, + ) + status = json.loads(out) + print("Replication status:") + print(json.dumps(status, indent=2)) + + +def teardown(cluster1, cluster2): + """ + Remove deployments from both clusters. This also deletes additonal + resources created in the same namespace. + """ + print("Cleaning up clusters") + subctl.unexport("service", VOLSYNC_SERVICE, cluster2, namespace=NAMESPACE) + kubectl.delete( + "--kustomize", + "source", + "--ignore-not-found", + "--wait=false", + context=cluster1, + ) + kubectl.delete( + "--kustomize", + "destination", + "--ignore-not-found", + "--wait=false", + context=cluster2, + ) + kubectl.delete("--kustomize", "source", "--ignore-not-found", context=cluster1) + kubectl.delete("--kustomize", "destination", "--ignore-not-found", context=cluster2) + + +if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} cluster1 cluster2") + sys.exit(1) + +os.chdir(os.path.dirname(__file__)) +cluster1, cluster2 = sys.argv[1:] + +setup_application(cluster1) +setup_replication_destination(cluster2) + +wait_for_application(cluster1) +wait_for_replication_destination(cluster2) + +setup_replication_secret(cluster1, cluster2) +setup_replication_service(cluster1, cluster2) + +run_replication(cluster1) + +teardown(cluster1, cluster2) diff --git a/test/envs/regional-dr.yaml b/test/envs/regional-dr.yaml index 728aea7b4..8e500aeec 100644 --- a/test/envs/regional-dr.yaml +++ b/test/envs/regional-dr.yaml @@ -18,6 +18,9 @@ templates: memory: "6g" extra_disks: 1 disk_size: "50g" + addons: + - volumesnapshots + - csi-hostpath-driver workers: - addons: - name: cert-manager @@ -33,7 +36,6 @@ templates: - name: olm - name: minio - name: velero - - name: volsync - name: "hub-cluster" driver: kvm2 container_runtime: containerd @@ -61,3 +63,6 @@ workers: - addons: - name: rbd-mirror args: ["dr1", "dr2"] + - addons: + - name: volsync + args: ["dr1", "dr2"] diff --git a/test/envs/volsync.yaml b/test/envs/volsync.yaml new file mode 100644 index 000000000..118818843 --- /dev/null +++ b/test/envs/volsync.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +# Enviroment for testing volsync deployment. +--- +name: volsync + +templates: + - name: hub + driver: kvm2 + container_runtime: containerd + network: default + cni: kindnet + memory: 2g + workers: + - addons: + - name: submariner + args: [hub, dr1, dr2] + - name: cluster + driver: kvm2 + container_runtime: containerd + network: default + cni: kindnet + memory: 4g + disk_size: 50g + addons: + - volumesnapshots + - csi-hostpath-driver + +profiles: + - name: hub + template: hub + - name: dr1 + template: cluster + - name: dr2 + template: cluster + +workers: + - addons: + - name: volsync + args: [dr1, dr2]