Add velero self test

The test deploy nginx, creates a backup, delete the namespace and store
nginx from the backup. Finally it cleans up so we don't leave anything
on the cluster.

The test is idempotent - if the tests fails and leave junk around, the
next run will delete backups and restores and start from scratch.

Issues:

- Restore completes successfully - but has a warning. The deployment
  looks correct and the warning looks like expected behavior when
  backing up all resources blindly.

    $ velero restore get
    NAME                          BACKUP         STATUS      STARTED                         COMPLETED                       ERRORS   WARNINGS   CREATED                         SELECTOR
    nginx-backup-20230602013503   nginx-backup   Completed   2023-06-02 01:35:03 +0300 IDT   2023-06-02 01:35:04 +0300 IDT   0        1          2023-06-02 01:35:03 +0300 IDT   <none>

    $ velero restore describe | grep Warn
    Warnings:
        nginx-example:  could not restore, Endpoints "my-nginx" already
        exists. Warning: the in-cluster version is different than the
        backed-up version.

    $ kubectl get all -n nginx-example
    NAME                                    READY   STATUS    RESTARTS   AGE
    pod/nginx-deployment-7c89967545-zgb7h   1/1     Running   0          41s
    pod/nginx-deployment-7c89967545-zrf2s   1/1     Running   0          41s

    NAME               TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
    service/my-nginx   LoadBalancer   10.103.206.98   <pending>     80:32334/TCP   41s

    NAME                               READY   UP-TO-DATE   AVAILABLE   AGE
    deployment.apps/nginx-deployment   2/2     2            2           41s

    NAME                                          DESIRED   CURRENT   READY   AGE
    replicaset.apps/nginx-deployment-7c89967545   2         2         2       41s

Signed-off-by: Nir Soffer <nsoffer@redhat.com>

test/addons/velero/credentials.conf

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = minio
+aws_secret_access_key = minio123

test/addons/velero/nginx.yaml

@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: 2017 the Velero contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Copied from velero-v1.11.0-linux-amd64/examples with changes:
+# - Add nginx label to the deploymnet - without it is not backed up.
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nginx-example
+  labels:
+    app: nginx
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  namespace: nginx-example
+  labels:
+    app: nginx
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - image: nginx:1.17.6
+        name: nginx
+        ports:
+        - containerPort: 80
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: nginx
+  name: my-nginx
+  namespace: nginx-example
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+  selector:
+    app: nginx
+  type: LoadBalancer

test/addons/velero/test

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: The RamenDR authors
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+import sys
+import time
+
+from drenv import kubectl
+from drenv import commands
+
+
+BACKUP = "nginx-backup"
+
+
+def test(cluster):
+    print("Deploying nginx example")
+    kubectl.apply("--filename", "nginx.yaml", context=cluster)
+
+    print("Waiting until nginx is rolled out")
+    kubectl.rollout(
+        "status",
+        "deploy/nginx-deployment",
+        "--namespace=nginx-example",
+        "--timeout=180s",
+        context=cluster,
+    )
+
+    print("Deleting older backups")
+    if get_backup(cluster, BACKUP):
+        delete_backup(cluster, BACKUP)
+
+    print("Backing up nginx example")
+    create_backup(cluster, BACKUP, "app=nginx")
+    print(describe_backup(cluster, BACKUP))
+
+    print("Simulating a disaster - deleting namespace nginx-example")
+    kubectl.delete("namespace", "nginx-example", context=cluster)
+
+    print("Restoring nginx example from backup")
+    restore_backup(cluster, BACKUP)
+    print(describe_restore(cluster))
+
+    print("Verifying nginx example deploymnet")
+    kubectl.rollout(
+        "status",
+        "deploy/nginx-deployment",
+        "--namespace=nginx-example",
+        "--timeout=60s",
+        context=cluster,
+    )
+
+    print("Deleting backup")
+    delete_backup(cluster, BACKUP)
+
+    print("Deleting nginx example")
+    kubectl.delete("--filename", "nginx.yaml", context=cluster)
+
+
+def get_backup(cluster, name):
+    out = commands.run(
+        "velero",
+        "backup",
+        "get",
+        "--output=json",
+        f"--kubecontext={cluster}",
+    )
+    info = json.loads(out)
+
+    # The response is not consistent, we get:
+    # - If there are no backups: BackupList with empty items list
+    # - If there is one backup: Backup
+    # - If there is more than one backuo: BackupList with items list of Backup objects.
+    if info["kind"] == "Backup":
+        backups = [info]
+    elif info["kind"] == "BackupList":
+        backups = info["items"]
+    else:
+        raise RuntimeError(f"Unexpected response: {info}")
+
+    for backup in backups:
+        if backup["metadata"]["name"] == name:
+            return backup
+
+
+def create_backup(cluster, name, selector):
+    print(f"Creating backup {name} with selector {selector}")
+    for line in commands.watch(
+        "velero",
+        "backup",
+        "create",
+        name,
+        f"--selector={selector}",
+        f"--kubecontext={cluster}",
+        "--wait",
+    ):
+        print(line)
+
+
+def delete_backup(cluster, name):
+    print(f"Deleting backup {name}")
+    for line in commands.watch(
+        "velero",
+        "backup",
+        "delete",
+        name,
+        "--confirm",
+        f"--kubecontext={cluster}",
+    ):
+        print(line)
+
+    print(f"Waiting until backup {name} is deleted")
+    start = time.monotonic()
+    delay = 0.125
+    while get_backup(cluster, name):
+        time.sleep(delay)
+        delay = min(2 * delay, 8)
+    elapsed = time.monotonic() - start
+    print(f"Backup {name} deleted in {elapsed:.3f} seconds")
+
+
+def describe_backup(cluster, name):
+    return commands.run(
+        "velero",
+        "backup",
+        "describe",
+        name,
+        f"--kubecontext={cluster}",
+    )
+
+
+def describe_restore(cluster):
+    return commands.run(
+        "velero",
+        "restore",
+        "describe",
+        f"--kubecontext={cluster}",
+    )
+
+
+def restore_backup(cluster, name):
+    print(f"Restoring {name} from backup")
+    for line in commands.watch(
+        "velero",
+        "restore",
+        "create",
+        f"--from-backup={name}",
+        f"--kubecontext={cluster}",
+        "--wait",
+    ):
+        print(line)
+
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} cluster")
+    sys.exit(1)
+
+os.chdir(os.path.dirname(__file__))
+cluster = sys.argv[1]
+test(cluster)