diff --git a/test/addons/submariner/base/dst/deploy.yaml b/test/addons/submariner/base/dst/deploy.yaml new file mode 100644 index 000000000..af6cd642c --- /dev/null +++ b/test/addons/submariner/base/dst/deploy.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: nginx +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx + name: nginx + ports: + - containerPort: 80 diff --git a/test/addons/submariner/base/dst/kustomization.yaml b/test/addons/submariner/base/dst/kustomization.yaml new file mode 100644 index 000000000..5acccdeda --- /dev/null +++ b/test/addons/submariner/base/dst/kustomization.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - namespace.yaml + - deploy.yaml + - service.yaml diff --git a/test/addons/submariner/base/dst/namespace.yaml b/test/addons/submariner/base/dst/namespace.yaml new file mode 100644 index 000000000..cd3989d55 --- /dev/null +++ b/test/addons/submariner/base/dst/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nginx diff --git a/test/addons/submariner/base/dst/service.yaml b/test/addons/submariner/base/dst/service.yaml new file mode 100644 index 000000000..4d2a53733 --- /dev/null +++ b/test/addons/submariner/base/dst/service.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: nginx +spec: + ports: + - port: 80 + selector: + app: nginx + type: ClusterIP diff --git a/test/addons/submariner/base/src/kustomization.yaml b/test/addons/submariner/base/src/kustomization.yaml new file mode 100644 index 000000000..70cd60803 --- /dev/null +++ b/test/addons/submariner/base/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - namespace.yaml + - pod.yaml diff --git a/test/addons/submariner/base/src/namespace.yaml b/test/addons/submariner/base/src/namespace.yaml new file mode 100644 index 000000000..cd3989d55 --- /dev/null +++ b/test/addons/submariner/base/src/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nginx diff --git a/test/addons/submariner/base/src/pod.yaml b/test/addons/submariner/base/src/pod.yaml new file mode 100644 index 000000000..a21fbbda7 --- /dev/null +++ b/test/addons/submariner/base/src/pod.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: test + namespace: nginx +spec: + containers: + - name: test + image: quay.io/submariner/nettest + command: + - sh + - -c + - | + trap exit TERM + sleep 300 & + wait diff --git a/test/addons/submariner/ns1/dst/kustomization.yaml b/test/addons/submariner/ns1/dst/kustomization.yaml new file mode 100644 index 000000000..348f4f7ed --- /dev/null +++ b/test/addons/submariner/ns1/dst/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/dst +namespace: ns1 diff --git a/test/addons/submariner/ns1/src/kustomization.yaml b/test/addons/submariner/ns1/src/kustomization.yaml new file mode 100644 index 000000000..d632fb907 --- /dev/null +++ b/test/addons/submariner/ns1/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/src +namespace: ns1 diff --git a/test/addons/submariner/ns2/dst/kustomization.yaml b/test/addons/submariner/ns2/dst/kustomization.yaml new file mode 100644 index 000000000..27e694cdc --- /dev/null +++ b/test/addons/submariner/ns2/dst/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/dst +namespace: ns2 diff --git a/test/addons/submariner/ns2/src/kustomization.yaml b/test/addons/submariner/ns2/src/kustomization.yaml new file mode 100644 index 000000000..bd9428b37 --- /dev/null +++ b/test/addons/submariner/ns2/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/src +namespace: ns2 diff --git a/test/addons/submariner/start b/test/addons/submariner/start index 434d2e0f2..e1b9c8731 100755 --- a/test/addons/submariner/start +++ b/test/addons/submariner/start @@ -5,7 +5,6 @@ import os import sys -import time import drenv from drenv import cluster as drenv_cluster diff --git a/test/addons/submariner/test b/test/addons/submariner/test index cc5f198c0..1d4add945 100755 --- a/test/addons/submariner/test +++ b/test/addons/submariner/test @@ -7,32 +7,218 @@ import os import sys import time +import drenv from drenv import commands +from drenv import kubectl from drenv import subctl +# We create "nginx" service on namepspace "ns1" on first cluster, and namespace +# "ns2" in the second cluster. The service is exported to the other cluster +# using the same namespace. +SERVICE = "nginx" +NS1 = "ns1" +NS2 = "ns2" -def wait_for_cluster(cluster, timeout=180): - delay = 1 - deadline = time.monotonic() + timeout - print("Waiting until subctl status is ok") +def wait_for_submariner(cluster, timeout=180): + start = time.monotonic() + deadline = start + timeout + delay = 1 while True: - print("Checking subctl status...") + print(f"Checking submariner status in cluster '{cluster}'") try: status = subctl.show("all", context=cluster) except commands.Error as e: - print(f"subctl failed with exitcode {e.exitcode}") + if time.monotonic() > deadline: + raise + + print(f"Check failed: {e}") + print(f"Retrying in {delay} seconds") + time.sleep(delay) + delay = min(delay * 2, 16) else: + print(f"Check completed in {time.monotonic() - start:.3f} seconds") print(status) break - if time.monotonic() > deadline: - raise RuntimeError("Timeout waiting for subctl status") - print(f"Retrying in {delay} seconds...") - time.sleep(delay) - delay = min(delay * 2, 16) +def deploy_service(namespace, dst, src): + """ + Deploy namespace resources in clusters. + """ + print(f"Deploying namespace '{namespace}' on clusters '{dst}'") + kubectl.apply(f"--kustomize={namespace}/dst", context=dst) + print(f"Deploying namespace '{namespace}' on clusters '{src}'") + kubectl.apply(f"--kustomize={namespace}/src", context=src) + + +def undeploy_service(namespace, dst, src): + """ + Undeploy namespace resources in clusters. + """ + print(f"Undeploying namespace '{namespace}' in cluster '{dst}'") + kubectl.delete( + f"--kustomize={namespace}/dst", + "--wait=false", + "--ignore-not-found", + context=dst, + ) + + print(f"Undeploying namespace '{namespace}' in cluster '{src}'") + kubectl.delete( + f"--kustomize={namespace}/src", + "--wait=false", + "--ignore-not-found", + context=src, + ) + + +def wait_for_delete(namespace, dst, src): + print(f"Waiting for deletion of namespace '{namespace}' in cluster '{dst}'") + kubectl.delete( + f"--kustomize={namespace}/dst", + "--wait=true", + "--ignore-not-found", + context=dst, + ) + + print(f"Waiting for deletion of namespace '{namespace}' in cluster '{src}'") + kubectl.delete( + f"--kustomize={namespace}/src", + "--wait=true", + "--ignore-not-found", + context=src, + ) + + +def wait_for_service(cluster, namespace): + print(f"Waiting until '{SERVICE}' is rolled out in cluster '{cluster}'") + kubectl.wait( + f"deploy/{SERVICE}", + "--for=condition=Available", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + + +def wait_for_pod(cluster, namespace): + print(f"Waiting until test pod is ready in cluster '{cluster}'") + kubectl.wait( + "pod/test", + "--for=condition=Ready", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + + +def export_service(cluster, namespace): + print(f"Export service in namespace '{namespace}' in cluster '{cluster}'") + subctl.export("service", SERVICE, cluster, namespace=namespace) + + +def unexport_service(cluster, namespace): + print(f"Unexport service in namespace '{namespace}' in cluster '{cluster}'") + subctl.unexport("service", SERVICE, cluster, namespace=namespace) + + +def wait_for_service_export(cluster, namespace): + print(f"Waiting for service in namespace '{namespace}' in cluster '{cluster}'") + kubectl.wait( + f"serviceexports/{SERVICE}", + "--for=condition=Synced=True", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + exports = kubectl.describe( + f"serviceexports/{SERVICE}", + f"--namespace={namespace}", + context=cluster, + ) + print(exports) + + +def wait_for_service_import(cluster, namespace): + print(f"Waiting for service in namespace '{namespace}' in cluster '{cluster}'") + drenv.wait_for( + f"serviceimports/{SERVICE}", + output="jsonpath={.status.clusters}", + namespace=namespace, + timeout=60, + profile=cluster, + ) + imports = kubectl.describe( + f"serviceimports/{SERVICE}", + f"--namespace={namespace}", + context=cluster, + ) + print(imports) + + +def service_address(namespace): + return f"{SERVICE}.{namespace}.svc.clusterset.local" + + +def wait_for_dns(cluster, namespace): + """ + Unfortunatley even when we wait for eveything, DNS lookup can fail for more + than 60 seconds after deploying submariner on a new cluster. After the + initial DNS always succceeds on the first try. + """ + start = time.monotonic() + deadline = start + 120 + dns_name = service_address(namespace) + delay = 1 + + while True: + print(f"Looking up '{dns_name}' in cluster '{cluster}'") + try: + out = kubectl.exec( + "test", + f"--namespace={namespace}", + "--", + "nslookup", + dns_name, + context=cluster, + ) + except commands.Error as e: + if time.monotonic() > deadline: + raise + + print(f"Lookup failed: {e}") + print(f"Retrying in {delay} seconds") + time.sleep(delay) + delay = min(delay * 2, 16) + else: + print(f"Lookup completed in {time.monotonic() - start:.3f} seconds") + print(out) + break + + +def test_connectivity(cluster, namespace): + """ + Test that cluster can access service exported on the other cluster. + + Unfortunatley even when we wait for eveything, DNS can fail so we must have + retries. + """ + dns_name = service_address(namespace) + + print(f"Accessing '{dns_name}' in cluster '{cluster}'") + out = kubectl.exec( + "test", + f"--namespace={namespace}", + "--", + "curl", + "--no-progress-meter", + dns_name, + context=cluster, + ) + if "Welcome to nginx" not in out: + raise RuntimeError(f"Unexpected output: {out}") if len(sys.argv) != 4: @@ -43,9 +229,38 @@ os.chdir(os.path.dirname(__file__)) broker = sys.argv[1] clusters = sys.argv[2:] -wait_for_cluster(broker) - +wait_for_submariner(broker) for cluster in clusters: - wait_for_cluster(cluster) + wait_for_submariner(cluster) + +deploy_service(NS1, clusters[0], clusters[1]) +deploy_service(NS2, clusters[1], clusters[0]) + +wait_for_service(clusters[0], NS1) +wait_for_pod(clusters[1], NS1) +wait_for_service(clusters[1], NS2) +wait_for_pod(clusters[0], NS2) + +export_service(clusters[0], NS1) +export_service(clusters[1], NS2) + +wait_for_service_export(clusters[0], NS1) +wait_for_service_import(clusters[1], NS1) + +wait_for_service_export(clusters[1], NS2) +wait_for_service_import(clusters[0], NS2) + +wait_for_dns(clusters[1], NS1) +test_connectivity(clusters[1], NS1) + +wait_for_dns(clusters[0], NS2) +test_connectivity(clusters[0], NS2) + +unexport_service(clusters[0], NS1) +unexport_service(clusters[1], NS2) + +undeploy_service(NS1, clusters[0], clusters[1]) +undeploy_service(NS2, clusters[1], clusters[0]) -test(cluster) +wait_for_delete(NS1, clusters[0], clusters[1]) +wait_for_delete(NS2, clusters[1], clusters[0]) diff --git a/test/drenv/kubectl.py b/test/drenv/kubectl.py index 147dec23e..91927c6dc 100644 --- a/test/drenv/kubectl.py +++ b/test/drenv/kubectl.py @@ -34,6 +34,10 @@ def get(*args, context=None): return _run("get", *args, context=context) +def describe(*args, context=None): + return _run("describe", *args, context=context) + + def exec(*args, context=None): """ Run kubectl get ... and return the output. diff --git a/test/drenv/subctl.py b/test/drenv/subctl.py index b1d768987..ceeef529f 100644 --- a/test/drenv/subctl.py +++ b/test/drenv/subctl.py @@ -35,6 +35,28 @@ def join(broker_info, context, clusterid, cable_driver=None, log=print): _watch(*args, log=log) +def export(what, name, context, namespace=None, log=print): + """ + Run subctl export ... logging progress messages. + """ + args = ["export", what, "--context", context] + if namespace: + args.extend(("--namespace", namespace)) + args.append(name) + _watch(*args, log=log) + + +def unexport(what, name, context, namespace=None, log=print): + """ + Run subctl unexport ... logging progress messages. + """ + args = ["unexport", what, "--context", context] + if namespace: + args.extend(("--namespace", namespace)) + args.append(name) + _watch(*args, log=log) + + def show(what, context): return commands.run("subctl", "show", what, "--context", context)