test: Sanity tests Selenium Grid chart via Makefile commands (#2029)

[deploy]
SeleniumHQ · Nov 28, 2023 · 7b3dfe9 · 7b3dfe9
1 parent e56a1fc
commit 7b3dfe9
Show file tree

Hide file tree

Showing 24 changed files with 358 additions and 230 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -12,18 +12,19 @@ jobs:
     # Skip job based on the commit message, only works in push to branches for now
     if: contains(toJson(github.event.commits), '[skip ci]') == false
     name: Build & test Docker images with random user
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         use-random-user: [false, true]
     steps:
       - uses: actions/checkout@v4
       - name: Output Docker info
         run: docker info
-      - name: Set up Python 3.8
+      - name: Set up Python
         uses: actions/setup-python@v4.7.1
         with:
-          python-version: 3.8
+          python-version: '3.11'
+          check-latest: true
       - name: Get branch name (only for push to branch)
         if: github.event_name == 'push'
         run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -10,7 +10,7 @@ jobs:
     # Only continue if the commit message has '[deploy]' in it
     if: contains(toJson(github.event.commits), '[deploy]') == true
     name: Deploy Docker images
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml
@@ -13,105 +13,47 @@ permissions:
   contents: read
 
 jobs:
-  lint-test:
-    name: "Lint Tests with ct"
+  build-and-test:
+    # Skip job based on the commit message, only works in push to branches for now
+    if: contains(toJson(github.event.commits), '[skip ci]') == false
+    name: Build & test Docker images with Helm charts
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.13.2
-
+      - uses: actions/checkout@v4
+      - name: Output Docker info
+        run: docker info
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v4.7.1
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           check-latest: true
-
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.6.1
-
-      - name: Run chart-testing (list-changed)
-        id: list-changed
-        run: |
-          changed=$(ct list-changed --config tests/chart-test.yaml)
-          if [[ -n "$changed" ]]; then
-            echo "{changed}={true}" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Run chart-testing (lint)
-        run: ct lint --config tests/chart-test.yaml
-
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.8.0
-        with:
-         config: ./tests/kind-cluster-config.yaml
-
-      - name: Run chart-testing (install)
-        run: ct install --all --config tests/chart-test.yaml
-
-  deploy-grid-selenium-tests:
-    name: "Run Selenium Tests on K8s"
-    runs-on: ubuntu-latest
-    strategy: 
-      fail-fast: false
-      matrix:
-        browser: [NodeChrome,NodeEdge,NodeFirefox]    
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.13.2
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-          check-latest: true
-
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.8.0
-        with:
-         config: ./tests/kind-cluster-config.yaml
-
-      # 👋 Documentation link for Ingress Installation on kind k8s cluster https://kind.sigs.k8s.io/docs/user/ingress
-      - name: Install ingress-nginx on kind kubernetes cluster
-        run: |
-          kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
-          kubectl wait --namespace ingress-nginx \
-          --for=condition=ready pod \
-          --selector=app.kubernetes.io/component=controller \
-          --timeout=90s
-
-      - name: Deploy Selenium Grid Chart 
-        run: |
-          helm repo add kedacore https://kedacore.github.io/charts
-          helm repo update
-          helm dependency build charts/selenium-grid          
-          helm upgrade --install selenium-grid -f ./tests/override-kind-auth-${{matrix.browser}}-values.yaml charts/selenium-grid --namespace selenium-grid-test --create-namespace
-          kubectl get ingress --all-namespaces
-
-      - name: Verify Post Deployment Grid Health and k8s pods status 
-        run: |
-          sleep 20 # Allow Kubernetes to pull Docker Images and start Pods
-          python ./tests/K8sSmokeTest.py "http://localhost"
-          kubectl get pods -n selenium-grid-test
-          kubectl get events -n selenium-grid-test
-
-      - name: Run Selenium Tests Against Kubernetes
-        run: |
-          export SELENIUM_GRID_HOST=localhost
-          export SELENIUM_GRID_PORT=80
-          export RUN_IN_DOCKER_COMPOSE=true
-          ./tests/bootstrap.sh ${{matrix.browser}}
+      - name: Get branch name (only for push to branch)
+        if: github.event_name == 'push'
+        run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV
+        env:
+          PUSH_BRANCH: ${{ github.ref }}
+      - name: Get target branch name (only for PRs)
+        if: github.event_name == 'pull_request'
+        run: echo "BRANCH=$(echo ${TARGET_BRANCH##*/})" >> $GITHUB_ENV
+        env:
+          TARGET_BRANCH: ${{ github.head_ref }}
+      - name: Output branch name
+        run: echo ${BRANCH}
+      - name: Sets build date
+        run: echo "BUILD_DATE=$(date '+%Y%m%d')" >> $GITHUB_ENV
+      - name: Build Docker images
+        run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
+      - name: Setup Kubernetes environment
+        run: make chart_setup_env
+      - name: Setup Kubernetes cluster
+        run: VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup
+      - name: Test Selenium Grid on Kubernetes
+        uses: nick-invision/retry@v2.9.0
+        with:
+          timeout_minutes: 20
+          max_attempts: 3
+          command: |
+            VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test
+      - name: Cleanup Kubernetes cluster
+        if: always()
+        run: make chart_cluster_cleanup
diff --git a/.github/workflows/label-commenter.yml b/.github/workflows/label-commenter.yml
@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   comment:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Label Commenter

diff --git a/.github/workflows/test-video.yml b/.github/workflows/test-video.yml
@@ -12,15 +12,16 @@ jobs:
     # Skip job based on the commit message, only works in push to branches for now
     if: contains(toJson(github.event.commits), '[skip ci]') == false
     name: Test video recorded through Docker Selenium
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Output Docker info
         run: docker info
-      - name: Set up Python 3.8
+      - name: Set up Python
         uses: actions/setup-python@v4.7.1
         with:
-          python-version: 3.8
+          python-version: '3.11'
+          check-latest: true
       - name: Get branch name (only for push to branch)
         if: github.event_name == 'push'
         run: echo "BRANCH=$(echo ${PUSH_BRANCH##*/})" >> $GITHUB_ENV
@@ -51,4 +52,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: firefox_video
-          path: ./tests/videos/firefox_video.mp4
+          path: ./tests/videos/firefox_video.mp4
diff --git a/Makefile b/Makefile
@@ -1,7 +1,7 @@
 NAME := $(or $(NAME),$(NAME),selenium)
 CURRENT_DATE := $(shell date '+%Y%m%d')
 BUILD_DATE := $(or $(BUILD_DATE),$(BUILD_DATE),$(CURRENT_DATE))
-VERSION := $(or $(VERSION),$(VERSION),4.11.0)
+VERSION := $(or $(VERSION),$(VERSION),4.15.0)
 TAG_VERSION := $(VERSION)-$(BUILD_DATE)
 NAMESPACE := $(or $(NAMESPACE),$(NAMESPACE),$(NAME))
 AUTHORS := $(or $(AUTHORS),$(AUTHORS),SeleniumHQ)
@@ -361,6 +361,32 @@ test_video: video hub chrome firefox edge
 	docker run -v $$(pwd):$$(pwd) -w $$(pwd) jrottenberg/ffmpeg:6.0-alpine -v error -i ./tests/videos/firefox_video.mp4 -f null - 2>error.log
 	docker run -v $$(pwd):$$(pwd) -w $$(pwd) jrottenberg/ffmpeg:6.0-alpine -v error -i ./tests/videos/edge_video.mp4 -f null - 2>error.log
 
+chart_setup_env:
+	./tests/K8s/chart_setup_env.sh
+
+chart_test: chart_lint \
+ chart_install_chrome \
+ chart_install_firefox \
+ chart_install_edge
+
+chart_cluster_setup:
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_cluster_setup.sh
+
+chart_lint:
+	./tests/K8s/chart_lint.sh
+
+chart_install_chrome:
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeChrome
+
+chart_install_firefox:
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeFirefox
+
+chart_install_edge:
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeEdge
+
+chart_cluster_cleanup:
+	./tests/K8s/chart_cluster_cleanup.sh
+
 .PHONY: \
 	all \
 	base \

diff --git a/charts/selenium-grid/TESTING.md b/charts/selenium-grid/TESTING.md
@@ -0,0 +1,44 @@
+# Testing Selenium Grid Helm Chart
+
+All related testing to this helm chart will be documented in this file.
+
+## Test Traceability Matrix
+
+| Features               | TC Description                                                       | Coverage |
+|------------------------|----------------------------------------------------------------------|----------|
+| Basic Auth             | Basic Auth is disabled                                               | &check;  |
+|                        | Basic Auth is enabled                                                | &cross;  |
+| Auto scaling           | Auto scaling with `enableWithExistingKEDA` is `true`                 | &check;  |
+|                        | Auto scaling with `scalingType` is `job`                             | &check;  |
+|                        | Auto scaling with `scalingType` is `deployment`                      | &cross;  |
+|                        | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | &check;  |
+| Ingress                | Ingress is enabled without `hostname`                                | &check;  |
+|                        | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths    | &check;  |
+| Distributed components | `isolateComponents` is enabled                                       | &check;  |
+| Browser Nodes          | Node `nameOverride` is set                                           | &check;  |
+|                        | Sanity tests in node                                                 | &check;  |
+|                        | Video recorder is enabled in node                                    | &cross;  |
+
+## Build & test Docker images with Helm charts
+Noted: These `make` commands are composed and tested on Linux x86_64.
+Run entire commands to build and test Docker images with Helm charts in local environment.
+
+```bash
+# Back to root directory
+cd ../..
+
+# Build Docker images
+make build
+
+# Setup Kubernetes environment
+make chart_setup_env
+
+# Setup Kubernetes cluster
+make chart_cluster_setup
+
+# Test Selenium Grid on Kubernetes
+make chart_test
+
+# Cleanup Kubernetes cluster
+make chart_cluster_cleanup
+```
diff --git a/...s/override-kind-auth-NodeEdge-values.yaml → ...s/selenium-grid/ci/NodeChrome-values.yaml b/...s/override-kind-auth-NodeEdge-values.yaml → ...s/selenium-grid/ci/NodeChrome-values.yaml
@@ -1,13 +1,10 @@
-# This is used in Helm chart testing. This disables the basic auth on seleneium grid
-# Basic auth settings for Selenium Grid
-basicAuth:
-  # Enable or disable basic auth
-  enabled: false
+# This is used in Helm chart testing. This disables the basic auth on selenium grid
 # Configuration for chrome nodes
 chromeNode:
-  # Enable chrome nodes
+  nameOverride: my-chrome-name
+# Configuration for edge nodes
+edgeNode:
   enabled: false
 # Configuration for firefox nodes
 firefoxNode:
-  # Enable firefox nodes
   enabled: false
diff --git a/...override-kind-auth-NodeChrome-values.yaml → charts/selenium-grid/ci/NodeEdge-values.yaml b/...override-kind-auth-NodeChrome-values.yaml → charts/selenium-grid/ci/NodeEdge-values.yaml
@@ -1,13 +1,10 @@
-# This is used in Helm chart testing. This disables the basic auth on seleneium grid
-# Basic auth settings for Selenium Grid
-basicAuth:
-  # Enable or disable basic auth
+# This is used in Helm chart testing. This disables the basic auth on selenium grid
+# Configuration for chrome nodes
+chromeNode:
   enabled: false
 # Configuration for edge nodes
 edgeNode:
-  # Enable edge nodes
-  enabled: false
+  nameOverride: my-edge-name
 # Configuration for firefox nodes
 firefoxNode:
-  # Enable firefox nodes
   enabled: false
diff --git a/...verride-kind-auth-NodeFirefox-values.yaml → .../selenium-grid/ci/NodeFirefox-values.yaml b/...verride-kind-auth-NodeFirefox-values.yaml → .../selenium-grid/ci/NodeFirefox-values.yaml
@@ -1,13 +1,10 @@
-# This is used in Helm chart testing. This disables the basic auth on seleneium grid
-# Basic auth settings for Selenium Grid
-basicAuth:
-  # Enable or disable basic auth
-  enabled: false
+# This is used in Helm chart testing. This disables the basic auth on selenium grid
 # Configuration for chrome nodes
 chromeNode:
-  # Enable chrome nodes
-  enabled: false  
+  enabled: false
 # Configuration for edge nodes
 edgeNode:
-  # Enable edge nodes
   enabled: false
+# Configuration for firefox nodes
+firefoxNode:
+  nameOverride: my-firefox-name
diff --git a/...ium-grid/ci/customIngressPath-values.yaml → ...selenium-grid/ci/auth-ingress-values.yaml b/...ium-grid/ci/customIngressPath-values.yaml → ...selenium-grid/ci/auth-ingress-values.yaml
@@ -6,14 +6,14 @@ ingress:
   hostname: ""
   paths:
     - path: /selenium(/|$)(.*)
-      pathType: Prefix
+      pathType: ImplementationSpecific
       backend:
         service:
           name: '{{ template "seleniumGrid.router.fullname" $ }}'
           port:
             number: 4444
     - path: /(/?)(session/.*/se/vnc)
-      pathType: Prefix
+      pathType: ImplementationSpecific
       backend:
         service:
           name: '{{ template "seleniumGrid.router.fullname" $ }}'
@@ -29,24 +29,3 @@ hub:
 
 components:
   subPath: *gridAppRoot
-
-chromeNode:
-  enabled: true
-  extraEnvironmentVariables:
-    - name: SE_NODE_OVERRIDE_MAX_SESSIONS
-      value: "true"
-    - name: SE_NODE_MAX_SESSIONS
-      value: "5"
-  startupProbe:
-    exec:
-      command:
-        - bash
-        - -c
-        - if [ $(curl --write-out %{http_code} --silent --output /dev/null http://selenium-router:4444/selenium/wd/hub/status) -ne 200 ]; then exit 1; fi
-    failureThreshold: 15
-    periodSeconds: 5
-
-edgeNode:
-  enabled: false
-firefoxNode:
-  enabled: false