From af16a26f192ced4e993834dcd72b12f1ece8f730 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Tue, 19 Dec 2023 15:50:07 +0530 Subject: [PATCH] feat(chart): Simplify to access Selenium Grid from outside of Kubernetes (#2073) [deploy] Signed-off-by: Viet Nguyen Duc --- charts/selenium-grid/Chart.yaml | 4 + charts/selenium-grid/README.md | 74 ++++++++++++++- charts/selenium-grid/templates/NOTES.txt | 19 ++-- charts/selenium-grid/templates/_helpers.tpl | 73 ++++++++++++-- .../templates/distributor-service.yaml | 3 + .../templates/event-bus-service.yaml | 9 ++ .../selenium-grid/templates/hub-service.yaml | 9 ++ .../templates/node-configmap.yaml | 2 +- .../templates/router-service.yaml | 3 + .../templates/session-queuer-service.yaml | 3 + charts/selenium-grid/values.yaml | 39 +++++++- .../charts/ci/ParallelAutoscaling-values.yaml | 7 -- tests/charts/ci/auth-ingress-values.yaml | 21 ++-- tests/charts/config/ct.yaml | 1 + tests/charts/make/chart_cluster_setup.sh | 7 -- tests/charts/refValues/simplex-minikube.yaml | 95 +++++++++++++++++++ tests/charts/templates/render/dummy.yaml | 19 ++++ tests/charts/templates/test.py | 13 ++- 18 files changed, 362 insertions(+), 39 deletions(-) create mode 100644 tests/charts/refValues/simplex-minikube.yaml diff --git a/charts/selenium-grid/Chart.yaml b/charts/selenium-grid/Chart.yaml index 6a0dbfdb6..1fedd8f1d 100644 --- a/charts/selenium-grid/Chart.yaml +++ b/charts/selenium-grid/Chart.yaml @@ -10,6 +10,10 @@ dependencies: version: 2.12.1 name: keda condition: autoscaling.enabled +- repository: https://kubernetes.github.io/ingress-nginx + version: 4.8.3 + name: ingress-nginx + condition: ingress-nginx.enabled maintainers: - name: SeleniumHQ email: selenium-developers@googlegroups.com diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 8681ed5e0..8b1f46225 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -2,6 +2,27 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes. +## Contents + +* [Selenium-Grid Helm Chart](#selenium-grid-helm-chart) + * [Contents](#contents) + * [Installing the chart](#installing-the-chart) + * [Enable Selenium Grid Autoscaling](#enable-selenium-grid-autoscaling) + * [Settings when scaling with deployments](#settings-when-scaling-with-deployments-) + * [Updating Selenium-Grid release](#updating-selenium-grid-release) + * [Uninstalling Selenium Grid release](#uninstalling-selenium-grid-release) + * [Ingress Configuration](#ingress-configuration) + * [References values](#references-values) + * [Configuration](#configuration) + * [Configuration global](#configuration-global) + * [Configuration `global.K8S_PUBLIC_IP`](#configuration-globalk8spublicip) + * [Configuration of Selenium Grid chart](#configuration-of-selenium-grid-chart) + * [Configuration of KEDA](#configuration-of-keda) + * [Configuration of Ingress NGINX Controller](#configuration-of-ingress-nginx-controller) + * [Configuration for Selenium-Hub](#configuration-for-selenium-hub) + * [Configuration for isolated components](#configuration-for-isolated-components) + + ## Installing the chart If you want to install the latest master version of Selenium Grid onto your cluster you can do that by using the helm charts repository located at https://www.selenium.dev/docker-selenium. @@ -127,12 +148,20 @@ nginx.ingress.kubernetes.io/client-body-buffer-size nginx.ingress.kubernetes.io/proxy-buffers-number ``` +## Reference values + +There are some values file that used to test and deploy Selenium Grid chart. You can find them in +- [tests/charts/refValues](../../tests/charts/refValues). +- [tests/charts/ci](../../tests/charts/ci). + ## Configuration +### Configuration global For now, global configuration supported is: | Parameter | Default | Description | |---------------------------------------|-----------------------|---------------------------------------| +| `global.K8S_PUBLIC_IP` | `""` | Public IP of the host running K8s | | `global.seleniumGrid.imageRegistry` | `selenium` | Distribution registry to pull images | | `global.seleniumGrid.imageTag` | `4.16.1-20231212` | Image tag for all selenium components | | `global.seleniumGrid.nodesImageTag` | `4.16.1-20231212` | Image tag for browser's nodes | @@ -142,6 +171,32 @@ For now, global configuration supported is: | `global.seleniumGrid.affinity` | `{}` | Affinity assigned globally | | `global.seleniumGrid.logLevel` | `INFO` | Set log level for all components | +#### Configuration `global.K8S_PUBLIC_IP` + +This is the public IP of the host running Kubernetes cluster. Mainly, it is used to construct the URL for the Selenium Grid (Hub or Router) can be accessed from the outside of the cluster for Node register, Grid UI, RemoteWebDriver, etc. +- Ingress is enabled without setting `ingress.hostname`. All the services will be exposed via the public IP is set in `K8S_PUBLIC_IP`. +- Using NodePort to expose the services. All the services will be exposed via the public IP is set in `K8S_PUBLIC_IP`. +- Using LoadBalancer to expose the services. All the services will be exposed via the LB External IP is set in `K8S_PUBLIC_IP`. + +For example: +```yaml +global: + K8S_PUBLIC_IP: "10.10.10.10" +ingress: + enabled: true + hostname: "" +hub: + subPath: "/selenium" + serviceType: NodePort +``` +``` +# Source: selenium-grid/templates/node-configmap.yaml + +SE_NODE_GRID_URL: 'http://admin:admin@10.10.10.10/selenium' +``` +Besides that, from the outside of the cluster, you can access via NodePort http://10.10.10.10:30444/selenium + +### Configuration of Selenium Grid chart This table contains the configuration parameters of the chart and their default values: | Parameter | Default | Description | @@ -163,6 +218,8 @@ This table contains the configuration parameters of the chart and their default | `ingress.nginx.proxyTimeout` | `3600` | Value is used to set for NGINX ingress annotations related to proxy timeout | | `ingress.nginx.proxyBuffer.size` | `512M` | Value is used to set for NGINX ingress annotations on size of the buffer proxy_buffer_size used for reading | | `ingress.nginx.proxyBuffer.number` | `4` | Value is used to set for NGINX ingress annotations on number of the buffers in proxy_buffers used for reading | +| `ingress.ports.http` | `80` | Port to expose for HTTP | +| `ingress.ports.https` | `443` | Port to expose for HTTPS | | `ingress.hostname` | `` | Default host for the ingress resource | | `ingress.path` | `/` | Default host path for the ingress resource | | `ingress.pathType` | `Prefix` | Default path type for the ingress resource | @@ -322,15 +379,21 @@ This table contains the configuration parameters of the chart and their default | `videoRecorder.s3.extraEnvFrom` | `` | Custom environment taken from `configMap` or `secret` variables for video uploader | | `videoRecorder.s3.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod of video uploader | | `customLabels` | `{}` | Custom labels for k8s resources | +| `ingress-nginx.enabled` | `false` | Enable the dependency chart Ingress controller for Kubernetes (https://github.com/kubernetes/ingress-nginx) | ### Configuration of KEDA -If you are setting `autoscaling.enabled` to `true` KEDA is installed and can be configured with +If you are setting `autoscaling.enabled` to `true`, chart KEDA is installed and can be configured with values with the prefix `keda`. So you can for example set `keda.prometheus.metricServer.enabled` to `true` to enable the metrics server for KEDA. See https://github.com/kedacore/charts/blob/main/keda/README.md for more details. +### Configuration of Ingress NGINX Controller + +If you are setting `ingress-nginx.enabled` to `true`, chart Ingress NGINX Controller is installed and can be configured with +values with the prefix `ingress-nginx`. See https://github.com/kubernetes/ingress-nginx for more details. + ### Configuration for Selenium-Hub You can configure the Selenium Hub with these values: @@ -345,8 +408,11 @@ You can configure the Selenium Hub with these values: | `hub.annotations` | `{}` | Custom annotations for Selenium Hub pod | | `hub.labels` | `{}` | Custom labels for Selenium Hub pod | | `hub.publishPort` | `4442` | Port where events are published | +| `hub.publishNodePort` | `31442` | NodePort where events are published | | `hub.subscribePort` | `4443` | Port where to subscribe for events | +| `hub.subscribeNodePort` | `31443` | NodePort where to subscribe for events | | `hub.port` | `4444` | Selenium Hub port | +| `hub.nodePort` | `31444` | Selenium Hub NodePort | | `hub.livenessProbe` | `See values.yaml` | Liveness probe settings | | `hub.readinessProbe` | `See values.yaml` | Readiness probe settings | | `hub.tolerations` | `[]` | Tolerations for selenium-hub pods | @@ -378,6 +444,7 @@ If you implement selenium-grid with separate components (`isolateComponents: tru | `components.router.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `components.router.annotations` | `{}` | Custom annotations for router pod | | `components.router.port` | `4444` | Router port | +| `components.router.nodePort` | `30444` | Router NodePort | | `components.router.livenessProbe` | `See values.yaml` | Liveness probe settings | | `components.router.readinessProbe` | `See values.yaml` | Readiness probe settings | | `components.router.resources` | `{}` | Resources for router pod | @@ -396,6 +463,7 @@ If you implement selenium-grid with separate components (`isolateComponents: tru | `components.distributor.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `components.distributor.annotations` | `{}` | Custom annotations for Distributor pod | | `components.distributor.port` | `5553` | Distributor port | +| `components.distributor.nodePort` | `30553` | Distributor NodePort | | `components.distributor.resources` | `{}` | Resources for Distributor pod | | `components.distributor.securityContext` | `See values.yaml` | Security context for Distributor pod | | `components.distributor.serviceType` | `ClusterIP` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | @@ -411,8 +479,11 @@ If you implement selenium-grid with separate components (`isolateComponents: tru | `components.eventBus.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `components.eventBus.annotations` | `{}` | Custom annotations for Event Bus pod | | `components.eventBus.port` | `5557` | Event Bus port | +| `components.eventBus.nodePort` | `30557` | Event Bus NodePort | | `components.eventBus.publishPort` | `4442` | Port where events are published | +| `components.eventBus.publishNodePort` | `30442` | NodePort where events are published | | `components.eventBus.subscribePort` | `4443` | Port where to subscribe for events | +| `components.eventBus.subscribeNodePort` | `30443` | NodePort where to subscribe for events | | `components.eventBus.resources` | `{}` | Resources for event-bus pod | | `components.eventBus.securityContext` | `See values.yaml` | Security context for event-bus pod | | `components.eventBus.serviceType` | `ClusterIP` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | @@ -442,6 +513,7 @@ If you implement selenium-grid with separate components (`isolateComponents: tru | `components.sessionQueue.imagePullSecret` | `""` | Image pull secret (see https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry) | | `components.sessionQueue.annotations` | `{}` | Custom annotations for Session Queue pod | | `components.sessionQueue.port` | `5559` | Session Queue Port | +| `components.sessionQueue.nodePort` | `30559` | Session Queue NodePort | | `components.sessionQueue.resources` | `{}` | Resources for Session Queue pod | | `components.sessionQueue.securityContext` | `See values.yaml` | Security context for Session Queue pod | | `components.sessionQueue.serviceType` | `ClusterIP` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | diff --git a/charts/selenium-grid/templates/NOTES.txt b/charts/selenium-grid/templates/NOTES.txt index ce79f74b5..949f854ec 100644 --- a/charts/selenium-grid/templates/NOTES.txt +++ b/charts/selenium-grid/templates/NOTES.txt @@ -9,20 +9,27 @@ Selenium Grid Server deployed successfully. {{- if .Values.ingress.enabled }} {{- if .Values.ingress.hostname }} 1. Ingress is enabled, and it exposes the Grid Hub or Grid Router with the hostname you supplied. - To access Selenium from outside of Kubernetes, simply open http://{{ .Values.ingress.hostname }}. - {{- else}} + To access Selenium from outside of Kubernetes, simply open {{ include "seleniumGrid.url" .}}. + {{- else if and (empty .Values.ingress.hostname) .Values.global.K8S_PUBLIC_IP }} +1. Ingress is enabled, but hostname doesn't set, and it exposes the Grid Hub or Grid Router with the K8S_PUBLIC_IP you supplied. + To access Selenium from outside of Kubernetes, simply open {{ include "seleniumGrid.url" .}}. + {{- else }} 1. Ingress is enabled, but hostname doesn't set. All inbound HTTP traffic will be routed to the Grid by matching any host. Please keep in mind that it is rarely necessary, and in most cases, you shall provide `ingress.hostname` in values.yaml. To access Selenium from outside of Kubernetes: - - open IP of the any node with Ingress, or + - open the IP of any node with Ingress, or - any hostname pointing to the node with Ingress {{- end}} {{- else}} -1. Ingress is disabled. To access Selenium from outside of Kubernetes, simply run these commands: {{- if contains "NodePort" $serviceType }} - export NODE_PORT=$(kubectl get -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" svc {{ $svcName }}) + {{- if .Values.global.K8S_PUBLIC_IP }} +1. Ingress is disabled, and it exposes the Grid Hub or Grid Router with NodePort and the K8S_PUBLIC_IP you supplied + To access Selenium from outside of Kubernetes with NodePort and K8S_PUBLIC_IP you supplied, simply open {{ include "seleniumGrid.url" .}}. + {{- else }} +1. Ingress is disabled. To access Selenium from outside of Kubernetes, simply run these commands: export NODE_IP=$(kubectl get nodes -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT + echo http://$NODE_IP:{{ include "seleniumGrid.url.port" .}}{{ include "seleniumGrid.url.subPath" .}} + {{- end }} {{- else if contains "LoadBalancer" $serviceType }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get -n {{ .Release.Namespace }} svc -w {{ $svcName }}' diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 12342ad9b..db7a8cedc 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -341,28 +341,87 @@ template: Get the url of the grid. If the external url can be figured out from the ingress use that, otherwise the cluster internal url */}} {{- define "seleniumGrid.url" -}} -{{- if and .Values.ingress.enabled .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}} -http{{if .Values.ingress.tls}}s{{end}}://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{.Values.ingress.hostname}} +{{- $url := printf "%s://%s%s%s%s" (include "seleniumGrid.url.schema" .) (include "seleniumGrid.url.basicAuth" .) (include "seleniumGrid.url.host" .) (include "seleniumGrid.url.port" .) (include "seleniumGrid.url.subPath" .) -}} +{{- $url }} +{{- end -}} + +{{- define "seleniumGrid.url.schema" -}} +{{- $schema := "http" -}} +{{- if .Values.ingress.enabled -}} + {{- if .Values.ingress.tls -}} + {{- $schema = "https" -}} + {{- end -}} +{{- end -}} +{{- $schema }} +{{- end -}} + +{{- define "seleniumGrid.url.basicAuth" -}} +{{- $basicAuth := "" -}} +{{- if eq .Values.basicAuth.enabled true -}} + {{- $basicAuth = printf "%s:%s@" .Values.basicAuth.username (.Values.basicAuth.password | toString) -}} +{{- end -}} +{{- $basicAuth }} +{{- end -}} + +{{- define "seleniumGrid.url.host" -}} +{{- $host := printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ ) (.Release.Namespace) -}} +{{- if .Values.ingress.enabled -}} + {{- if and ( empty .Values.ingress.hostname) (not (empty .Values.global.K8S_PUBLIC_IP)) -}} + {{- $host = .Values.global.K8S_PUBLIC_IP -}} + {{- else if and .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}} + {{- $host = .Values.ingress.hostname -}} + {{- end -}} +{{- else if not (empty .Values.global.K8S_PUBLIC_IP) -}} + {{- $host = .Values.global.K8S_PUBLIC_IP -}} +{{- end -}} +{{- $host }} +{{- end -}} + +{{- define "seleniumGrid.url.port" -}} +{{- $port := ":4444" -}} +{{- if .Values.ingress.enabled -}} + {{- if or (ne (.Values.ingress.ports.http | toString) "80") (ne (.Values.ingress.ports.https | toString) "443") -}} + {{- $port = printf ":%s" (ternary (.Values.ingress.ports.http | toString) (.Values.ingress.ports.https | toString) (eq (include "seleniumGrid.url.schema" .) "http")) -}} + {{- else -}} + {{- $port = "" -}} + {{- end -}} {{- else -}} -http://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }} -{{- end }} + {{- if .Values.isolateComponents -}} + {{- if and (eq .Values.components.router.serviceType "NodePort") .Values.components.router.nodePort -}} + {{- $port = printf ":%s" (.Values.components.router.nodePort | toString) -}} + {{- end -}} + {{- else -}} + {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.nodePort -}} + {{- $port = printf ":%s" (.Values.hub.nodePort | toString) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- $port }} {{- end -}} {{- define "seleniumGrid.url.subPath" -}} {{- $subPath := "/" -}} -{{ if $.Values.isolateComponents }} +{{- if $.Values.isolateComponents -}} {{- $subPath = default $subPath $.Values.components.subPath -}} {{- else -}} {{- $subPath = default $subPath $.Values.hub.subPath -}} {{- end -}} -{{ $subPath }} +{{- $subPath }} {{- end -}} {{/* Graphql Url of the hub or the router */}} {{- define "seleniumGrid.graphqlURL" -}} -http://{{- if eq .Values.basicAuth.enabled true}}{{ .Values.basicAuth.username}}:{{ .Values.basicAuth.password}}@{{- end}}{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}/graphql +{{- printf "http://%s%s%s/graphql" (include "seleniumGrid.url.basicAuth" .) (printf "%s.%s" (include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $) (.Release.Namespace)) (printf ":%s" ($.Values.isolateComponents | ternary ($.Values.components.router.port | toString) ($.Values.hub.port | toString))) -}} +{{- end -}} + +{{/* +Graphql unsafeSsl of the hub or the router +*/}} +{{- define "seleniumGrid.graphqlURL.unsafeSsl" -}} +{{- $unsafeSsl := printf "%s" (ternary "false" "true" (contains (include "seleniumGrid.graphqlURL" .) "https")) -}} +{{- $unsafeSsl }} {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/distributor-service.yaml b/charts/selenium-grid/templates/distributor-service.yaml index 295407fac..86b99e5b4 100644 --- a/charts/selenium-grid/templates/distributor-service.yaml +++ b/charts/selenium-grid/templates/distributor-service.yaml @@ -23,4 +23,7 @@ spec: protocol: TCP port: {{ .Values.components.distributor.port }} targetPort: {{ .Values.components.distributor.port }} + {{- if and (eq .Values.components.distributor.serviceType "NodePort") .Values.components.distributor.nodePort }} + nodePort: {{ .Values.components.distributor.nodePort }} + {{- end }} {{- end }} diff --git a/charts/selenium-grid/templates/event-bus-service.yaml b/charts/selenium-grid/templates/event-bus-service.yaml index c3a30bcc0..c5c5b9052 100644 --- a/charts/selenium-grid/templates/event-bus-service.yaml +++ b/charts/selenium-grid/templates/event-bus-service.yaml @@ -23,12 +23,21 @@ spec: protocol: TCP port: {{ .Values.components.eventBus.port }} targetPort: {{ .Values.components.eventBus.port }} + {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.nodePort }} + nodePort: {{ .Values.components.eventBus.nodePort }} + {{- end }} - name: tcp-evtbus-pub protocol: TCP port: {{ .Values.components.eventBus.publishPort }} targetPort: {{ .Values.components.eventBus.publishPort }} + {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.publishNodePort }} + nodePort: {{ .Values.components.eventBus.publishNodePort }} + {{- end }} - name: tcp-evtbus-sub protocol: TCP port: {{ .Values.components.eventBus.subscribePort }} targetPort: {{ .Values.components.eventBus.subscribePort }} + {{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.subscribeNodePort }} + nodePort: {{ .Values.components.eventBus.subscribeNodePort }} + {{- end }} {{- end }} diff --git a/charts/selenium-grid/templates/hub-service.yaml b/charts/selenium-grid/templates/hub-service.yaml index aa4daa34c..0f394a6eb 100644 --- a/charts/selenium-grid/templates/hub-service.yaml +++ b/charts/selenium-grid/templates/hub-service.yaml @@ -26,12 +26,21 @@ spec: protocol: TCP port: {{ .Values.hub.port }} targetPort: {{ .Values.hub.port }} + {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.nodePort }} + nodePort: {{ .Values.hub.nodePort }} + {{- end }} - name: tcp-hub-pub protocol: TCP port: {{ .Values.hub.publishPort }} targetPort: {{ .Values.hub.publishPort }} + {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.publishNodePort }} + nodePort: {{ .Values.hub.publishNodePort }} + {{- end }} - name: tcp-hub-sub protocol: TCP port: {{ .Values.hub.subscribePort }} targetPort: {{ .Values.hub.subscribePort }} + {{- if and (eq .Values.hub.serviceType "NodePort") .Values.hub.subscribeNodePort }} + nodePort: {{ .Values.hub.subscribeNodePort }} + {{- end }} {{- end }} diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index 52e447d63..da4edab40 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -13,4 +13,4 @@ metadata: {{- end }} data: SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' - SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}{{ include "seleniumGrid.url.subPath" .}}' + SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' diff --git a/charts/selenium-grid/templates/router-service.yaml b/charts/selenium-grid/templates/router-service.yaml index 9eaff7955..c93d04c11 100644 --- a/charts/selenium-grid/templates/router-service.yaml +++ b/charts/selenium-grid/templates/router-service.yaml @@ -26,4 +26,7 @@ spec: protocol: TCP port: {{ .Values.components.router.port }} targetPort: {{ .Values.components.router.port }} + {{- if and (eq $.Values.components.router.serviceType "NodePort") $.Values.components.router.nodePort }} + nodePort: {{ $.Values.components.router.nodePort }} + {{- end }} {{- end }} diff --git a/charts/selenium-grid/templates/session-queuer-service.yaml b/charts/selenium-grid/templates/session-queuer-service.yaml index 54e535f7b..b0e0fe96d 100644 --- a/charts/selenium-grid/templates/session-queuer-service.yaml +++ b/charts/selenium-grid/templates/session-queuer-service.yaml @@ -23,4 +23,7 @@ spec: protocol: TCP port: {{ .Values.components.sessionQueue.port }} targetPort: {{ .Values.components.sessionQueue.port }} + {{- if and (eq .Values.components.sessionQueue.serviceType "NodePort") .Values.components.sessionQueue.nodePort }} + nodePort: {{ .Values.components.sessionQueue.nodePort }} + {{- end }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index f6be10a83..c7b46c536 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1,4 +1,8 @@ global: + # Public IP of the host running Kubernetes cluster. + # This is used to access the Selenium Grid from outside the cluster when ingress is disabled or enabled without a hostname is set. + # This is part of constructing SE_NODE_GRID_URL and rewrite URL of `se:vnc`, `se:cdp` in the capabilities when `ingress.hostname` is unset + K8S_PUBLIC_IP: "" seleniumGrid: # Image registry for all selenium components imageRegistry: selenium @@ -44,6 +48,9 @@ ingress: proxyBuffer: size: 512M number: 4 + ports: + http: 80 + https: 443 # Custom annotations for ingress resource annotations: {} # Default host for the ingress resource @@ -96,6 +103,7 @@ components: annotations: {} # Router port port: 4444 + nodePort: 30444 # Liveness probe settings livenessProbe: enabled: true @@ -148,6 +156,7 @@ components: annotations: {} # Distributor port port: 5553 + nodePort: 30553 # Resources for Distributor container resources: {} # SecurityContext for Distributor container @@ -180,10 +189,13 @@ components: annotations: {} # Event Bus port port: 5557 + nodePort: 30557 # Port where events are published publishPort: 4442 + publishNodePort: 30442 # Port where to subscribe for events subscribePort: 4443 + subscribeNodePort: 30443 # Resources for event-bus container resources: {} # SecurityContext for event-bus container @@ -246,6 +258,7 @@ components: # Custom annotations for Session Queue pods annotations: {} port: 5559 + nodePort: 30559 # Resources for Session Queue container resources: {} # SecurityContext for Session Queue container @@ -299,10 +312,13 @@ hub: labels: {} # Port where events are published publishPort: 4442 + publishNodePort: 31442 # Port where to subscribe for events subscribePort: 4443 + subscribeNodePort: 31443 # Selenium Hub port port: 4444 + nodePort: 31444 # Liveness probe settings livenessProbe: enabled: true @@ -387,7 +403,8 @@ autoscaling: # see https://keda.sh/docs/latest/concepts/scaling-jobs/#scaledjob-spec scaledJobOptions: scalingStrategy: - strategy: accurate + # Change this to "accurate" when the calculation problem is fixed + strategy: default # Number of Completed jobs should be kept successfulJobsHistoryLimit: 0 # Number of Failed jobs should be kept (for troubleshooting purposes) @@ -535,8 +552,9 @@ chromeNode: hpa: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: chrome + sessionBrowserName: 'chrome' # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid. - unsafeSsl: 'true' # Optional + unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. @@ -668,6 +686,8 @@ firefoxNode: hpa: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: firefox + sessionBrowserName: 'firefox' + unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. @@ -798,6 +818,7 @@ edgeNode: url: '{{ include "seleniumGrid.graphqlURL" . }}' browserName: MicrosoftEdge sessionBrowserName: 'msedge' + unsafeSsl: '{{ include "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional # It is used to add a sidecars proxy in the same pod of the browser node. # It means it will add a new container to the deployment itself. @@ -918,3 +939,17 @@ videoRecorder: # Custom labels for k8s resources customLabels: {} + +# Configuration for dependency chart keda +keda: + http: + timeout: 60000 + webhooks: + enabled: false + +# Configuration for dependency chart ingress-nginx +ingress-nginx: + enabled: false + controller: + admissionWebhooks: + enabled: false diff --git a/tests/charts/ci/ParallelAutoscaling-values.yaml b/tests/charts/ci/ParallelAutoscaling-values.yaml index 39ea9434c..78302b94f 100644 --- a/tests/charts/ci/ParallelAutoscaling-values.yaml +++ b/tests/charts/ci/ParallelAutoscaling-values.yaml @@ -31,10 +31,3 @@ ingress: name: '{{ template "seleniumGrid.hub.fullname" $ }}' port: number: 4444 - - path: /(/?)(session/.*/se/vnc) - pathType: ImplementationSpecific - backend: - service: - name: '{{ template "seleniumGrid.hub.fullname" $ }}' - port: - number: 4444 diff --git a/tests/charts/ci/auth-ingress-values.yaml b/tests/charts/ci/auth-ingress-values.yaml index e5d055ba6..38bc87e1b 100644 --- a/tests/charts/ci/auth-ingress-values.yaml +++ b/tests/charts/ci/auth-ingress-values.yaml @@ -1,11 +1,16 @@ +global: + K8S_PUBLIC_IP: localhost + ingress: annotations: + kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" nginx.ingress.kubernetes.io/proxy-read-timeout: "360" nginx.ingress.kubernetes.io/proxy-send-timeout: "360" + ingressClassName: nginx hostname: "" paths: - path: /selenium(/|$)(.*) @@ -15,16 +20,10 @@ ingress: name: '{{ template "seleniumGrid.router.fullname" $ }}' port: number: 4444 - - path: /(/?)(session/.*/se/vnc) - pathType: ImplementationSpecific - backend: - service: - name: '{{ template "seleniumGrid.router.fullname" $ }}' - port: - number: 4444 basicAuth: enabled: false + isolateComponents: true hub: @@ -32,3 +31,11 @@ hub: components: subPath: *gridAppRoot + +ingress-nginx: + enabled: true + controller: + hostNetwork: true + kind: DaemonSet + service: + type: ClusterIP diff --git a/tests/charts/config/ct.yaml b/tests/charts/config/ct.yaml index 7c085d076..2b2446823 100755 --- a/tests/charts/config/ct.yaml +++ b/tests/charts/config/ct.yaml @@ -5,6 +5,7 @@ chart-dirs: - charts chart-repos: - kedacore=https://kedacore.github.io/charts + - ingressNginx=https://kubernetes.github.io/ingress-nginx upgrade: false helm-extra-args: --timeout 600s check-version-increment: false diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index 76b27872d..c87cbc5b1 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -38,13 +38,6 @@ kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests echo "Install KEDA core on kind kubernetes cluster" kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.12.1/keda-2.12.1-core.yaml -echo "Install ingress-nginx on kind kubernetes cluster" -kubectl apply -n ${INGRESS_NAMESPACE} -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml -kubectl wait --namespace ${INGRESS_NAMESPACE} \ - --for=condition=ready pod \ - --selector=app.kubernetes.io/component=controller \ - --timeout=${WAIT_TIMEOUT} - echo "Load built local Docker Images into Kind Cluster" image_list=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep ${NAMESPACE} | grep ${VERSION}) for image in $image_list; do diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml new file mode 100644 index 000000000..eb6100336 --- /dev/null +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -0,0 +1,95 @@ +# README: This is a sample values for chart deployment in Minikube +# Chart dependency ingress-nginx is installed together by enabling `ingress-nginx.enabled` +# Chart dependency keda is installed together by enabling `autoscaling.enable` +# Enabled ingress without hostname, set the subPath `/selenium`. Set K8S_PUBLIC_IP point to the public host IP, where Minikube is running +# `ingress-nginx.controller.hostNetwork` is set to true to allow access from outside the cluster via http:///selenium +# Components serviceType is set to NodePort to allow access from outside the cluster via K8S_PUBLIC_IP and NodePort http://:30444/selenium +global: + K8S_PUBLIC_IP: "10.10.10.10" # Replace with your public IP + seleniumGrid: + logLevel: INFO +# imageRegistry: selenium +# imageTag: latest +# nodesImageTag: latest +# videoImageTag: latest + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" + ingressClassName: nginx + hostname: "" + paths: + - path: /selenium(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: '{{ template "seleniumGrid.router.fullname" $ }}' + port: + number: 4444 + +basicAuth: + enabled: true + +isolateComponents: true + +autoscaling: + enabled: true + scalingType: job + annotations: + helm.sh/hook: post-install,post-upgrade,post-rollback + scaledOptions: + minReplicaCount: 0 + maxReplicaCount: 8 + pollingInterval: 15 + scaledJobOptions: + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 5 + scalingStrategy: + strategy: default + +hub: + subPath: *gridAppRoot + serviceType: NodePort + +components: + subPath: *gridAppRoot + router: + serviceType: NodePort + +chromeNode: + extraEnvironmentVariables: &extraEnvironmentVariablesNodes + - name: SE_NODE_SESSION_TIMEOUT + value: "300" + - name: SE_VNC_NO_PASSWORD + value: "true" + - name: SE_OPTS + value: "--enable-managed-downloads true" + startupProbe: &nodeStartupProbe + httpGet: + path: /status + port: 5555 + failureThreshold: 120 + periodSeconds: 1 + +firefoxNode: + extraEnvironmentVariables: *extraEnvironmentVariablesNodes + startupProbe: *nodeStartupProbe + +edgeNode: + extraEnvironmentVariables: *extraEnvironmentVariablesNodes + startupProbe: *nodeStartupProbe + +videoRecorder: + enabled: false + +ingress-nginx: + enabled: true + controller: + hostNetwork: true + kind: DaemonSet + service: + type: ClusterIP diff --git a/tests/charts/templates/render/dummy.yaml b/tests/charts/templates/render/dummy.yaml index fbb746166..42864d78f 100644 --- a/tests/charts/templates/render/dummy.yaml +++ b/tests/charts/templates/render/dummy.yaml @@ -1,5 +1,6 @@ # This is dummy values file for chart template testing global: + K8S_PUBLIC_IP: "10.10.10.10" seleniumGrid: logLevel: FINE affinity: &affinity @@ -12,6 +13,11 @@ global: values: - selenium topologyKey: "kubernetes.io/hostname" + +basicAuth: + username: sysadmin + password: strongPassword + ingress: nginx: proxyTimeout: 360 # Set different proxy timout @@ -25,6 +31,9 @@ ingress: nginx.ingress.kubernetes.io/proxy-connect-timeout: "3600" # Override default key nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" # Override default key hostname: "" + ports: + http: 8081 + https: 8443 paths: - path: /selenium(/|$)(.*) pathType: ImplementationSpecific @@ -45,6 +54,16 @@ isolateComponents: true components: subPath: *gridAppRoot + router: + serviceType: NodePort + distributor: + serviceType: NodePort + eventBus: + serviceType: NodePort + sessionQueue: + serviceType: NodePort + sessionMap: + serviceType: NodePort chromeNode: affinity: *affinity diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py index 2f0c2f412..8587b42d7 100644 --- a/tests/charts/templates/test.py +++ b/tests/charts/templates/test.py @@ -51,7 +51,7 @@ def test_sub_path_append_to_node_grid_url(self): for doc in LIST_OF_DOCUMENTS: if doc['metadata']['name'] in resources_name and doc['kind'] == 'ConfigMap': logger.info(f"Assert subPath is appended to node grid url") - self.assertTrue(doc['data']['SE_NODE_GRID_URL'] == 'http://admin:admin@selenium-router.default:4444/selenium') + self.assertTrue(doc['data']['SE_NODE_GRID_URL'] == 'http://sysadmin:strongPassword@10.10.10.10:8081/selenium') count += 1 self.assertEqual(count, len(resources_name), "No node config resources found") @@ -90,6 +90,17 @@ def test_log_level_set_to_logging_config_map(self): count += 1 self.assertEqual(count, len(resources_name), "Logging ConfigMap is not present in expected resources") + def test_node_port_set_when_service_type_is_node_port(self): + single_node_port = {'selenium-distributor': 30553, 'selenium-router': 30444, 'selenium-session-queue': 30559} + count = 0 + logger.info(f"Assert NodePort is set to components service") + for doc in LIST_OF_DOCUMENTS: + if doc['metadata']['name'] in single_node_port.keys() and doc['kind'] == 'Service': + logger.info(f"Assert NodePort is set to service {doc['metadata']['name']}") + self.assertTrue(doc['spec']['ports'][0]['nodePort'] == single_node_port[doc['metadata']['name']], f"Service {doc['metadata']['name']} with expect NodePort {single_node_port[doc['metadata']['name']]} is not found") + count += 1 + self.assertEqual(count, len(single_node_port.keys()), "Number of services with NodePort is not correct") + if __name__ == '__main__': failed = False try: