Parallelize Drone testing (#233)

* Parallelize drone testing * Reduce test load * Convert unit tests to github action Signed-off-by: Derek Nola <derek.nola@suse.com>
k3s-io · Nov 1, 2023 · 13bff3e · 13bff3e
1 parent 7ae068c
commit 13bff3e
Show file tree

Hide file tree

Showing 8 changed files with 128 additions and 18 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -18,24 +18,98 @@ steps:
   - name: docker
     path: /var/run/docker.sock
 
-- name: test
-  image: rancher/dapper:v0.6.0
+- name: test-image
+  image: rancher/dapper:v0.5.0
   commands:
   - docker build -t kine:test-${DRONE_COMMIT} -f Dockerfile.test .
-  - >
-   docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
-   -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
-   --privileged kine:test-${DRONE_COMMIT}
   environment:
     ARCH: "${DRONE_STAGE_ARCH}"
-    DOCKER_BUILDKIT: "1"
   volumes:
   - name: docker
     path: /var/run/docker.sock
   when:
     instance:
     - drone-pr.k3s.io
 
+- name: test-sqlite
+  image: rancher/dapper:v0.5.0
+  depends_on:
+   - build
+   - test-image
+  commands:
+  - > 
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test sqlite"
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test-mysql
+  image: rancher/dapper:v0.5.0
+  depends_on:
+   - build
+   - test-image
+  commands:
+  - > 
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test mysql"
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test-postgres
+  image: rancher/dapper:v0.5.0
+  depends_on:
+   - build
+   - test-image
+  commands:
+  - > 
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test postgres"
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test-cockroachdb
+  image: rancher/dapper:v0.5.0
+  depends_on:
+   - build
+   - test-image
+  commands:
+  - > 
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test cockroachdb"
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+- name: test-nats
+  image: rancher/dapper:v0.5.0
+  depends_on:
+   - build
+   - test-image
+  commands:
+  - > 
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test nats"
+  - >
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test nats-embedded"
+  - >
+    docker run -i -e ARCH -e REPO -e TAG  -e DRONE_TAG -e IMAGE_NAME
+    -v /var/run/docker.sock:/var/run/docker.sock -v kine-cache:/go/src/github.com/k3s-io/kine/.cache
+    --privileged kine:test-${DRONE_COMMIT} "./scripts/test nats-socket"
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+
+
 - name: github_binary_release
   image: plugins/github-release
   settings:

diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -0,0 +1,19 @@
+name: Unit Test
+on:
+  push:
+    branches: [main, master]
+
+  pull_request:
+    branches: [main, master]
+
+permissions:
+  contents: read
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v4
+    - name: Go Unit Tests
+      run: go test -cover -tags=test ./...
diff --git a/Dockerfile.test b/Dockerfile.test
@@ -12,5 +12,5 @@ WORKDIR ${KINE_SOURCE}
 
 COPY . ${KINE_SOURCE}
 
-ENTRYPOINT ["./scripts/entry"]
-CMD ["test"]
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["./scripts/test"]
diff --git a/hack/histogram.py b/hack/histogram.py
@@ -69,7 +69,7 @@ def main(type, *args, **kwargs):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--type', '-t', type=str, required=False, default='core.ConfigMap')
+    parser.add_argument('--type', '-t', type=str, required=False, default='configmaps')
     parser.add_argument('--log-level', '-l', type=str, required=False, default='INFO')
     parser.add_argument('--backend-name', '-b', type=str, required=False, default='etcd')
     args = parser.parse_args()

diff --git a/hack/loadmap.py b/hack/loadmap.py
@@ -162,7 +162,7 @@ def list_configmaps():
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--rounds', '-r', type=int, required=False, default=5000)
+    parser.add_argument('--rounds', '-r', type=int, required=False, default=4000)
     parser.add_argument('--log-level', '-l', type=str, required=False, default='INFO')
     args = parser.parse_args()
 

diff --git a/scripts/test b/scripts/test
@@ -5,15 +5,20 @@ cd $(dirname $0)/..
 
 . ./scripts/test-helpers
 
-#echo Running go unit tests
-go test -cover -tags=test ./...
-
 # ---
 
 docker ps
 
 # ---
 
+# Selectively run tests
+if [ -n "$1" ]; then
+    . ./scripts/test-run-"$1"
+    echo "Did test-run-$1 $?"
+    exit 0
+fi
+
+
 . ./scripts/test-run-sqlite
 echo "Did test-run-sqlite $?"
 

diff --git a/scripts/test-helpers b/scripts/test-helpers
@@ -124,6 +124,18 @@ export -f inc-count
 
 # ---
 
+find-free-port() {
+    activePorts=$(docker ps -a --format "{{.Names}}\t{{.Ports}}" | awk -F  ' ' '{if($2){print $2}}')
+    for port in $(shuf -i 5000-8000 -n 20); do
+        if ! echo "$activePorts" | grep -q "$port"; then
+            echo "$port"
+            return 0
+        fi
+    done
+}
+export -f find-free-port
+# ---
+
 has-function() {
   [[ ! -z "$1" && $(type -t $1) == "function" ]]
 } 2> /dev/null
@@ -196,13 +208,14 @@ provision-server() {
     local count=$(inc-count servers)
     local testID=$(basename $TEST_DIR)
     local name=$(echo "k3s-server-$count-$testID" | tee $TEST_DIR/servers/$count/metadata/name)
+    local testPort=$(find-free-port)
 
     run-function server-pre-hook $count
 
     docker container run \
         -d --name $name \
         --privileged \
-        -p 6443:6443 \
+        -p $testPort:6443 \
         -e K3S_DEBUG=true \
         -e K3S_DATASTORE_ENDPOINT=$K3S_DATASTORE_ENDPOINT \
         ${K3S_IMAGE:-docker.io/rancher/k3s:v1.26.9-k3s1} server \
@@ -213,7 +226,7 @@ provision-server() {
     local port=$(docker container inspect --format '{{range $k, $v := .NetworkSettings.Ports}}{{printf "%s\n" $k}}{{end}}' $name | head -n 1 | cut -d/ -f1 | tee $TEST_DIR/servers/$count/metadata/port)
     local url=$(echo "https://$ip:$port" | tee $TEST_DIR/servers/$count/metadata/url)
 
-    echo "Started $name @ $url"
+    echo "Started $name @ $url (hp:$testPort)"
     run-function server-post-hook $count
 }
 export -f provision-server

diff --git a/scripts/test-run-postgres b/scripts/test-run-postgres
@@ -26,8 +26,7 @@ VERSION_LIST="\
     postgres 15.4
     postgres 14.9
     postgres 13.12
-    postgres 12.16
-    postgres 11.21-alpine" # A generic 11.21 image is not available
+    postgres 12.16"
 
 while read ENGINE VERSION; do
     LABEL=$ENGINE-$VERSION DB_PASSWORD_ENV=POSTGRES_PASSWORD DB_IMAGE=docker.io/library/$ENGINE:$VERSION run-test