From c4085237bb369433d3720c8734f3d7ebe8e61c6b Mon Sep 17 00:00:00 2001
From: Nir Soffer <nsoffer@redhat.com>
Date: Thu, 19 Sep 2024 21:05:15 +0300
Subject: [PATCH] Run drenv cluster tests

On macos-13 runner we can run the test cluster using nested
virtualization. Build and install lima and socket_vmnet, and add the
missing steps to start the test cluster and run all the tests, including
cluster tests.

We may want to make the kubernetes ready timeout configurable, since
locally it completes in few seconds, but it time outs with 10 minutes
timeout on github action.

Signed-off-by: Nir Soffer <nsoffer@redhat.com>
---
 .github/workflows/ci.yaml          | 75 ++++++++++++++++++++++++++++--
 test/drenv/providers/lima/k8s.yaml |  4 +-
 2 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f58fcf08b..74ad0aabb 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -186,8 +186,8 @@ jobs:
     strategy:
       matrix:
         os:
-          # arm64, 3 cpus (M1), 7 GiB RAM, no nested virtalization.
-          - macos-14
+          # amd64, 4 cpus, 14 GiB RAM, nested virtalization
+          - macos-13
         python-version:
           - "3.12"
     runs-on: ${{ matrix.os }}
@@ -195,11 +195,58 @@ jobs:
       - name: Checkout ramen
         uses: actions/checkout@v4
 
+      - name: Checkout lima
+        uses: actions/checkout@v4
+        with:
+          repository: lima-vm/lima
+          path: lima
+
+      - name: Checkout socket_vmnet
+        uses: actions/checkout@v4
+        with:
+          repository: lima-vm/socket_vmnet
+          path: socket_vmnet
+
       - name: Setup python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Setup go for lima
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.23"  # used by lima-ci
+
+      - name: Install lima
+        run: |
+          cd lima
+
+          # Miniaml build is 3 times faster than full make..
+          make minimal
+
+          # But it adds only the default template, so lets fix it:
+          rm -rf _output/share/lima/templates
+          cp -aL examples _output/share/lima/templates
+          ln -sf templates _output/share/lima/examples
+
+          # And install
+          sudo make install
+          limactl --version
+
+      - name: Install socket_vmnet
+        run: |
+          cd socket_vmnet
+          make
+          sudo make install.bin
+          sudo make install.launchd
+          /opt/socket_vmnet/bin/socket_vmnet --version
+
+      - name: Allow bootpd to receive incoming connections
+        run: |
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --add /usr/libexec/bootpd
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --unblock /usr/libexec/bootpd
+          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --list
+
       - name: Install Homebrew packages
         run: brew install kubectl
 
@@ -224,15 +271,35 @@ jobs:
         run: make black
         working-directory: test
 
+      # Strting the cluster takes more than 10 minutes.
+      - name: Start test cluster
+        run: make cluster TIMEOUT=1200
+        working-directory: test
+        timeout-minutes: 20
+
+      - name: Show logs on error
+        if: failure()
+        run: |
+          export DIR=$(limactl list --format '{{.Dir}}')
+          # Don't fail entire step if one log is missing.
+          for f in ha.stdout.log ha.stderr.log copied-from-guest/kubectl.yaml; do
+            echo
+            echo "=== $DIR/$f ==="
+            cat $DIR/$f || true
+          done
+
       - name: Run tests
-        # We cannot run the test cluster on macOS.
-        run: make unit-test
+        run: make test
         working-directory: test
 
       - name: Report test coverage
         run: make coverage
         working-directory: test
 
+      - name: Clean up
+        run: make clean
+        working-directory: test
+
   ramenctl:
     name: ramenctl
     strategy:
diff --git a/test/drenv/providers/lima/k8s.yaml b/test/drenv/providers/lima/k8s.yaml
index a063e0052..5600be21f 100644
--- a/test/drenv/providers/lima/k8s.yaml
+++ b/test/drenv/providers/lima/k8s.yaml
@@ -198,7 +198,9 @@ probes:
     script: |
       #!/bin/bash
       set -eux -o pipefail
-      if ! timeout 600s bash -c "until kubectl get --raw /readyz 2>/dev/null; do sleep 3; done"; then
+      # Using 15m to make this work on github actions. When running locally
+      # this check complets in few seconds.
+      if ! timeout 15m bash -c "until kubectl get --raw /readyz 2>/dev/null; do sleep 3; done"; then
         echo >&2 "kubernetes cluster is not ready yet"
         exit 1
       fi