From 9774ed7b14ab2a775ef7f1cd5367312df8cc8ed6 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 21 Jun 2023 02:21:49 +0300 Subject: [PATCH] Test submariner connectivity Add a real connectivity test, creating nginx deployment on one cluster, exporting the service using submariner, and accessing nginx from the second cluster. We run the test in both ways, deploying on dr1 and accessing from dr2, and deploying on dr2 and accessing from dr1. Both tests run concurrently by separating deploy and wait steps. The entire test takes 15 seconds, but on the first run after deploying there is more than 60 seconds delay before service DNS name can be resolved, and the test takes 85 seconds. We don't see any delay when running on the regional-dr environment. Example run with submariner.yaml: $ drenv start envs/submariner.yaml 2023-06-21 18:27:00,108 INFO [submariner] Starting environment 2023-06-21 18:27:00,135 INFO [hub] Starting minikube cluster 2023-06-21 18:27:01,165 INFO [dr1] Starting minikube cluster 2023-06-21 18:27:02,160 INFO [dr2] Starting minikube cluster 2023-06-21 18:27:38,610 INFO [hub] Cluster started in 38.47 seconds 2023-06-21 18:27:38,611 INFO [hub/0] Running addons/submariner/start 2023-06-21 18:27:58,850 INFO [dr1] Cluster started in 57.69 seconds 2023-06-21 18:28:20,728 INFO [dr2] Cluster started in 78.57 seconds 2023-06-21 18:29:10,467 INFO [hub/0] addons/submariner/start completed in 91.86 seconds 2023-06-21 18:29:10,467 INFO [hub/0] Running addons/submariner/test 2023-06-21 18:30:35,047 INFO [hub/0] addons/submariner/test completed in 84.58 seconds 2023-06-21 18:30:35,048 INFO [submariner] Environment started in 214.94 seconds Example run with regional-dr.yaml: $ drenv start envs/regional-dr.yaml 2023-06-21 18:54:05,517 INFO [rdr] Starting environment ... 2023-06-21 18:55:27,894 INFO [hub/1] Running addons/submariner/start ... 2023-06-21 18:59:42,084 INFO [hub/1] addons/submariner/start completed in 254.19 seconds 2023-06-21 18:59:42,084 INFO [hub/1] Running addons/submariner/test ... 2023-06-21 19:00:10,954 INFO [hub/1] addons/submariner/test completed in 28.87 seconds 2023-06-21 19:01:47,030 INFO [rdr] Environment started in 461.51 seconds Signed-off-by: Nir Soffer --- test/addons/submariner/base/dst/deploy.yaml | 24 ++ .../submariner/base/dst/kustomization.yaml | 8 + .../addons/submariner/base/dst/namespace.yaml | 8 + test/addons/submariner/base/dst/service.yaml | 15 ++ .../submariner/base/src/kustomization.yaml | 7 + .../addons/submariner/base/src/namespace.yaml | 8 + test/addons/submariner/base/src/pod.yaml | 20 ++ .../submariner/ns1/dst/kustomization.yaml | 7 + .../submariner/ns1/src/kustomization.yaml | 7 + .../submariner/ns2/dst/kustomization.yaml | 7 + .../submariner/ns2/src/kustomization.yaml | 7 + test/addons/submariner/start | 1 - test/addons/submariner/test | 244 ++++++++++++++++-- test/drenv/kubectl.py | 4 + test/drenv/subctl.py | 22 ++ 15 files changed, 373 insertions(+), 16 deletions(-) create mode 100644 test/addons/submariner/base/dst/deploy.yaml create mode 100644 test/addons/submariner/base/dst/kustomization.yaml create mode 100644 test/addons/submariner/base/dst/namespace.yaml create mode 100644 test/addons/submariner/base/dst/service.yaml create mode 100644 test/addons/submariner/base/src/kustomization.yaml create mode 100644 test/addons/submariner/base/src/namespace.yaml create mode 100644 test/addons/submariner/base/src/pod.yaml create mode 100644 test/addons/submariner/ns1/dst/kustomization.yaml create mode 100644 test/addons/submariner/ns1/src/kustomization.yaml create mode 100644 test/addons/submariner/ns2/dst/kustomization.yaml create mode 100644 test/addons/submariner/ns2/src/kustomization.yaml diff --git a/test/addons/submariner/base/dst/deploy.yaml b/test/addons/submariner/base/dst/deploy.yaml new file mode 100644 index 0000000000..af6cd642cd --- /dev/null +++ b/test/addons/submariner/base/dst/deploy.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: nginx +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx + name: nginx + ports: + - containerPort: 80 diff --git a/test/addons/submariner/base/dst/kustomization.yaml b/test/addons/submariner/base/dst/kustomization.yaml new file mode 100644 index 0000000000..5acccdeda2 --- /dev/null +++ b/test/addons/submariner/base/dst/kustomization.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - namespace.yaml + - deploy.yaml + - service.yaml diff --git a/test/addons/submariner/base/dst/namespace.yaml b/test/addons/submariner/base/dst/namespace.yaml new file mode 100644 index 0000000000..cd3989d556 --- /dev/null +++ b/test/addons/submariner/base/dst/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nginx diff --git a/test/addons/submariner/base/dst/service.yaml b/test/addons/submariner/base/dst/service.yaml new file mode 100644 index 0000000000..4d2a537339 --- /dev/null +++ b/test/addons/submariner/base/dst/service.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: nginx +spec: + ports: + - port: 80 + selector: + app: nginx + type: ClusterIP diff --git a/test/addons/submariner/base/src/kustomization.yaml b/test/addons/submariner/base/src/kustomization.yaml new file mode 100644 index 0000000000..70cd608032 --- /dev/null +++ b/test/addons/submariner/base/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - namespace.yaml + - pod.yaml diff --git a/test/addons/submariner/base/src/namespace.yaml b/test/addons/submariner/base/src/namespace.yaml new file mode 100644 index 0000000000..cd3989d556 --- /dev/null +++ b/test/addons/submariner/base/src/namespace.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nginx diff --git a/test/addons/submariner/base/src/pod.yaml b/test/addons/submariner/base/src/pod.yaml new file mode 100644 index 0000000000..a21fbbda7c --- /dev/null +++ b/test/addons/submariner/base/src/pod.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: v1 +kind: Pod +metadata: + name: test + namespace: nginx +spec: + containers: + - name: test + image: quay.io/submariner/nettest + command: + - sh + - -c + - | + trap exit TERM + sleep 300 & + wait diff --git a/test/addons/submariner/ns1/dst/kustomization.yaml b/test/addons/submariner/ns1/dst/kustomization.yaml new file mode 100644 index 0000000000..348f4f7ede --- /dev/null +++ b/test/addons/submariner/ns1/dst/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/dst +namespace: ns1 diff --git a/test/addons/submariner/ns1/src/kustomization.yaml b/test/addons/submariner/ns1/src/kustomization.yaml new file mode 100644 index 0000000000..d632fb9074 --- /dev/null +++ b/test/addons/submariner/ns1/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/src +namespace: ns1 diff --git a/test/addons/submariner/ns2/dst/kustomization.yaml b/test/addons/submariner/ns2/dst/kustomization.yaml new file mode 100644 index 0000000000..27e694cdce --- /dev/null +++ b/test/addons/submariner/ns2/dst/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/dst +namespace: ns2 diff --git a/test/addons/submariner/ns2/src/kustomization.yaml b/test/addons/submariner/ns2/src/kustomization.yaml new file mode 100644 index 0000000000..bd9428b376 --- /dev/null +++ b/test/addons/submariner/ns2/src/kustomization.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +--- +resources: + - ../../base/src +namespace: ns2 diff --git a/test/addons/submariner/start b/test/addons/submariner/start index 434d2e0f2a..e1b9c87313 100755 --- a/test/addons/submariner/start +++ b/test/addons/submariner/start @@ -5,7 +5,6 @@ import os import sys -import time import drenv from drenv import cluster as drenv_cluster diff --git a/test/addons/submariner/test b/test/addons/submariner/test index cc5f198c00..9c327053de 100755 --- a/test/addons/submariner/test +++ b/test/addons/submariner/test @@ -7,32 +7,217 @@ import os import sys import time +import drenv from drenv import commands +from drenv import kubectl from drenv import subctl +# We create "nginx" service on namepspace "ns1" on first cluster, and namespace +# "ns2" in the second cluster. The service is exported to the other cluster +# using the same namespace. +SERVICE = "nginx" +NS1 = "ns1" +NS2 = "ns2" -def wait_for_cluster(cluster, timeout=180): - delay = 1 - deadline = time.monotonic() + timeout - print("Waiting until subctl status is ok") +def wait_for_submariner(cluster, timeout=180): + start = time.monotonic() + deadline = start + timeout + delay = 1 while True: - print("Checking subctl status...") + print(f"Checking submariner status in cluster '{cluster}'") try: status = subctl.show("all", context=cluster) except commands.Error as e: - print(f"subctl failed with exitcode {e.exitcode}") + if time.monotonic() > deadline: + raise + + print(f"Check failed: {e}") + print(f"Retrying in {delay} seconds") + time.sleep(delay) + delay = min(delay * 2, 16) else: + print(f"Check completed in {time.monotonic() - start:.3f} seconds") print(status) break - if time.monotonic() > deadline: - raise RuntimeError("Timeout waiting for subctl status") - print(f"Retrying in {delay} seconds...") - time.sleep(delay) - delay = min(delay * 2, 16) +def deploy_service(namespace, dst, src): + """ + Deploy namespace resources in clusters. + """ + print(f"Deploying namespace '{namespace}' on clusters '{dst}'") + kubectl.apply(f"--kustomize={namespace}/dst", context=dst) + print(f"Deploying namespace '{namespace}' on clusters '{src}'") + kubectl.apply(f"--kustomize={namespace}/src", context=src) + + +def undeploy_service(namespace, dst, src): + """ + Undeploy namespace resources in clusters. + """ + print(f"Undeploying namespace '{namespace}' in cluster '{dst}'") + kubectl.delete( + f"--kustomize={namespace}/dst", + "--wait=false", + "--ignore-not-found", + context=dst, + ) + + print(f"Undeploying namespace '{namespace}' in cluster '{src}'") + kubectl.delete( + f"--kustomize={namespace}/src", + "--wait=false", + "--ignore-not-found", + context=src, + ) + + +def wait_for_delete(namespace, dst, src): + print(f"Waiting for deletion of namespace '{namespace}' in cluster '{dst}'") + kubectl.delete( + f"--kustomize={namespace}/dst", + "--wait=true", + "--ignore-not-found", + context=dst, + ) + + print(f"Waiting for deletion of namespace '{namespace}' in cluster '{src}'") + kubectl.delete( + f"--kustomize={namespace}/src", + "--wait=true", + "--ignore-not-found", + context=src, + ) + + +def wait_for_service(cluster, namespace): + print(f"Waiting until '{SERVICE}' is rolled out in cluster '{cluster}'") + kubectl.wait( + f"deploy/{SERVICE}", + "--for=condition=Available", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + + +def wait_for_pod(cluster, namespace): + print(f"Waiting until test pod is ready in cluster '{cluster}'") + kubectl.wait( + "pod/test", + "--for=condition=Ready", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + + +def export_service(cluster, namespace): + print(f"Export service in namespace '{namespace}' in cluster '{cluster}'") + subctl.export("service", SERVICE, cluster, namespace=namespace) + + +def unexport_service(cluster, namespace): + print(f"Unexport service in namespace '{namespace}' in cluster '{cluster}'") + subctl.unexport("service", SERVICE, cluster, namespace=namespace) + + +def wait_for_service_export(cluster, namespace): + print(f"Waiting for service in namespace '{namespace}' in cluster '{cluster}'") + kubectl.wait( + f"serviceexports/{SERVICE}", + "--for=condition=Synced=True", + f"--namespace={namespace}", + "--timeout=60s", + context=cluster, + ) + exports = kubectl.describe( + f"serviceexports/{SERVICE}", + f"--namespace={namespace}", + context=cluster, + ) + print(exports) + + +def wait_for_service_import(cluster, namespace): + print(f"Waiting for service in namespace '{namespace}' in cluster '{cluster}'") + drenv.wait_for( + f"serviceimports/{SERVICE}", + output="jsonpath={.status.clusters}", + namespace=namespace, + timeout=60, + profile=cluster, + ) + imports = kubectl.describe( + f"serviceimports/{SERVICE}", + f"--namespace={namespace}", + context=cluster, + ) + print(imports) + + +def service_address(namespace): + return f"{SERVICE}.{namespace}.svc.clusterset.local" + + +def wait_for_dns(cluster, namespace): + """ + Unfortunatley even when we wait for eveything, DNS lookup fails in the + first XXX seconds after deploying submariner on a new cluster. + """ + start = time.monotonic() + deadline = start + 60 + dns_name = service_address(namespace) + delay = 1 + + while True: + print(f"Looking up '{dns_name}' in cluster '{cluster}'") + try: + out = kubectl.exec( + "test", + f"--namespace={namespace}", + "--", + "nslookup", + dns_name, + context=cluster, + ) + except commands.Error as e: + if time.monotonic() > deadline: + raise + + print(f"Lookup failed: {e}") + print(f"Retrying in {delay} seconds") + time.sleep(delay) + delay = min(delay * 2, 16) + else: + print(f"Lookup completed in {time.monotonic() - start:.3f} seconds") + print(out) + break + + +def test_connectivity(cluster, namespace): + """ + Test that cluster can access service exported on the other cluster. + + Unfortunatley even when we wait for eveything, DNS can fail so we must have + retries. + """ + dns_name = service_address(namespace) + + print(f"Accessing '{dns_name}' in cluster '{cluster}'") + out = kubectl.exec( + "test", + f"--namespace={namespace}", + "--", + "curl", + "--no-progress-meter", + dns_name, + context=cluster, + ) + if "Welcome to nginx" not in out: + raise RuntimeError(f"Unexpected output: {out}") if len(sys.argv) != 4: @@ -43,9 +228,38 @@ os.chdir(os.path.dirname(__file__)) broker = sys.argv[1] clusters = sys.argv[2:] -wait_for_cluster(broker) - +wait_for_submariner(broker) for cluster in clusters: - wait_for_cluster(cluster) + wait_for_submariner(cluster) + +deploy_service(NS1, clusters[0], clusters[1]) +deploy_service(NS2, clusters[1], clusters[0]) + +wait_for_service(clusters[0], NS1) +wait_for_pod(clusters[1], NS1) +wait_for_service(clusters[1], NS2) +wait_for_pod(clusters[0], NS2) + +export_service(clusters[0], NS1) +export_service(clusters[1], NS2) + +wait_for_service_export(clusters[0], NS1) +wait_for_service_import(clusters[1], NS1) + +wait_for_service_export(clusters[1], NS2) +wait_for_service_import(clusters[0], NS2) + +wait_for_dns(clusters[1], NS1) +test_connectivity(clusters[1], NS1) + +wait_for_dns(clusters[0], NS2) +test_connectivity(clusters[0], NS2) + +unexport_service(clusters[0], NS1) +unexport_service(clusters[1], NS2) + +undeploy_service(NS1, clusters[0], clusters[1]) +undeploy_service(NS2, clusters[1], clusters[0]) -test(cluster) +wait_for_delete(NS1, clusters[0], clusters[1]) +wait_for_delete(NS2, clusters[1], clusters[0]) diff --git a/test/drenv/kubectl.py b/test/drenv/kubectl.py index 147dec23ec..91927c6dc9 100644 --- a/test/drenv/kubectl.py +++ b/test/drenv/kubectl.py @@ -34,6 +34,10 @@ def get(*args, context=None): return _run("get", *args, context=context) +def describe(*args, context=None): + return _run("describe", *args, context=context) + + def exec(*args, context=None): """ Run kubectl get ... and return the output. diff --git a/test/drenv/subctl.py b/test/drenv/subctl.py index b1d768987c..ceeef529f1 100644 --- a/test/drenv/subctl.py +++ b/test/drenv/subctl.py @@ -35,6 +35,28 @@ def join(broker_info, context, clusterid, cable_driver=None, log=print): _watch(*args, log=log) +def export(what, name, context, namespace=None, log=print): + """ + Run subctl export ... logging progress messages. + """ + args = ["export", what, "--context", context] + if namespace: + args.extend(("--namespace", namespace)) + args.append(name) + _watch(*args, log=log) + + +def unexport(what, name, context, namespace=None, log=print): + """ + Run subctl unexport ... logging progress messages. + """ + args = ["unexport", what, "--context", context] + if namespace: + args.extend(("--namespace", namespace)) + args.append(name) + _watch(*args, log=log) + + def show(what, context): return commands.run("subctl", "show", what, "--context", context)