From 56f3c4976509984ccd86f3a4b91725a019fe3def Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 19 Feb 2024 12:31:36 +0700 Subject: [PATCH] update(chart): Node preStop and startupProbe in autoscaling Deployment (#2139) update(chart): script for Node preStop and startupProbe in autoscaling Deployment Signed-off-by: Viet Nguyen Duc --- .github/workflows/helm-chart-test.yml | 11 +++- Makefile | 12 +++- Video/video.sh | 5 +- .../selenium-grid/configs/node/nodePreStop.sh | 62 ++++++++++++++++++- .../selenium-grid/configs/node/nodeProbe.sh | 41 ++++++++++++ charts/selenium-grid/templates/_helpers.tpl | 30 ++++----- .../templates/hub-deployment.yaml | 12 ++-- .../templates/node-configmap.yaml | 11 +++- .../templates/router-deployment.yaml | 12 ++-- charts/selenium-grid/values.yaml | 19 ++++-- .../ci/DeploymentAutoscaling-values.yaml | 2 - tests/charts/ci/JobAutoscaling-values.yaml | 2 - tests/charts/ci/base-resources-values.yaml | 18 ++++++ tests/charts/make/chart_test.sh | 13 ++-- 14 files changed, 201 insertions(+), 49 deletions(-) create mode 100644 charts/selenium-grid/configs/node/nodeProbe.sh diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 55886a7b3..c1c96417c 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -15,6 +15,9 @@ on: description: 'Test parameter for different request timeout' required: false default: '370' + cluster: + description: 'Cluster type to test (kind, minikube)' + required: false schedule: - cron: '0 0 * * *' @@ -33,6 +36,8 @@ jobs: - job_https - deployment - deployment_https + env: + CLUSTER: ${{ github.event.inputs.cluster || 'minikube' }} steps: - uses: actions/checkout@main - name: Output Docker info @@ -63,7 +68,7 @@ jobs: with: timeout_minutes: 10 max_attempts: 3 - command: make chart_setup_env + command: CLUSTER=${CLUSTER} make chart_setup_env - name: Build Helm charts run: | BUILD_DATE=${BUILD_DATE} make chart_build @@ -76,7 +81,7 @@ jobs: with: timeout_minutes: 10 max_attempts: 3 - command: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup + command: CLUSTER=${CLUSTER} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_cluster_setup - name: Build Docker images run: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build - name: Test set custom CA certificate @@ -96,7 +101,7 @@ jobs: NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make chart_test_autoscaling_${{ matrix.test-strategy }} - name: Cleanup Kubernetes cluster if: always() - run: make chart_cluster_cleanup + run: CLUSTER=${CLUSTER} make chart_cluster_cleanup - name: Upload Helm chart package if: always() uses: actions/upload-artifact@main diff --git a/Makefile b/Makefile index c1635fdae..8314863d8 100644 --- a/Makefile +++ b/Makefile @@ -447,16 +447,22 @@ chart_test_edge: ./tests/charts/make/chart_test.sh NodeEdge chart_test_autoscaling_deployment_https: - SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_deployment + SE_FULL_DISTRIBUTED_MODE=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_deployment: - SE_ENABLE_TRACING=true VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh DeploymentAutoscaling chart_test_autoscaling_job_https: - SE_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 make chart_test_autoscaling_job + SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ + ./tests/charts/make/chart_test.sh JobAutoscaling chart_test_autoscaling_job: + SE_ENABLE_TRACING=true SE_FULL_DISTRIBUTED_MODE=true \ VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \ ./tests/charts/make/chart_test.sh JobAutoscaling diff --git a/Video/video.sh b/Video/video.sh index 4d7ddf812..ca06910a3 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -159,7 +159,10 @@ else echo Checking if node API responds until curl -sk --request GET ${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT}/status || [[ $attempts = "$max_attempts" ]] do - echo Waiting before next API check + if [ $(($attempts % 60)) -eq 0 ]; + then + echo Waiting before next API check + fi sleep 0.5 attempts=$((attempts+1)) done diff --git a/charts/selenium-grid/configs/node/nodePreStop.sh b/charts/selenium-grid/configs/node/nodePreStop.sh index af7d32ba9..d030d8e26 100644 --- a/charts/selenium-grid/configs/node/nodePreStop.sh +++ b/charts/selenium-grid/configs/node/nodePreStop.sh @@ -1,4 +1,11 @@ #!/bin/bash + +function on_exit() { + rm -rf /tmp/preStopOutput +} +trap on_exit EXIT + +# Set headers if Node Registration Secret is set if [ ! -z "${SE_REGISTRATION_SECRET}" ]; then HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}" @@ -6,8 +13,61 @@ else HEADERS="X-REGISTRATION-SECRET;" fi -if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status; then +function is_full_distributed_mode() { + if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then + DISTRIBUTED_MODE=true + echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap" + else + DISTRIBUTED_MODE=false + echo "Detected full distributed mode: ${DISTRIBUTED_MODE}" + fi +} +is_full_distributed_mode + +function signal_distributor_to_drain_node() { + if [ "${DISTRIBUTED_MODE}" = true ]; then + echo "Signaling Distributor to drain node" + set -x + curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + set +x + fi +} + +function signal_hub_to_drain_node() { + if [ "${DISTRIBUTED_MODE}" = false ]; then + echo "Signaling Hub to drain node" + curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}" + fi +} + +function signal_node_to_drain() { + echo "Signaling Node to drain itself" curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}" +} + +function replace_localhost_by_service_name() { + internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" + if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} + fi + echo "SE_GRID_URL: ${SE_GRID_URL}" +} +replace_localhost_by_service_name + +if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then + NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput) + if [ -n "${NODE_ID}" ]; then + echo "Current Node ID is: ${NODE_ID}" + signal_hub_to_drain_node + signal_distributor_to_drain_node + echo + fi + signal_node_to_drain + # Wait for the current session to be finished if any while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput; do echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")" diff --git a/charts/selenium-grid/configs/node/nodeProbe.sh b/charts/selenium-grid/configs/node/nodeProbe.sh new file mode 100644 index 000000000..236afad5b --- /dev/null +++ b/charts/selenium-grid/configs/node/nodeProbe.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +function on_exit() { + rm -rf /tmp/nodeProbe${ID} + rm -rf /tmp/gridProbe${ID} +} +trap on_exit EXIT + +ID=$(echo $RANDOM) + +function replace_localhost_by_service_name() { + internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}" + if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}} + elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then + SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}} + fi + echo "SE_GRID_URL: ${SE_GRID_URL}" +} +replace_localhost_by_service_name + +if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then + NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID}) + NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID}) + + curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID} + GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true) + + if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then + echo "Node ID: ${NODE_ID} with status: ${NODE_STATUS}" + echo "Found in the Grid a matched Node ID: ${GRID_NODE_ID}" + exit 0 + else + echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress." + exit 1 + fi +else + exit 1 +fi diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 62a56e904..ebd15ad38 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -17,18 +17,20 @@ Probe httpGet schema Check user define custom probe method */}} {{- define "seleniumGrid.probe.fromUserDefine" -}} +{{- $values := index . "values" -}} +{{- $root := index . "root" -}} {{- $overrideProbe := dict -}} -{{- with .exec -}} -{{- $overrideProbe = dict "exec" . -}} +{{- with $values.exec -}} +{{- $overrideProbe = dict "exec" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .httpGet -}} -{{- $overrideProbe = dict "httpGet" . -}} +{{- with $values.httpGet -}} +{{- $overrideProbe = dict "httpGet" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .tcpSocket -}} -{{- $overrideProbe = dict "tcpSocket" . -}} +{{- with $values.tcpSocket -}} +{{- $overrideProbe = dict "tcpSocket" (tpl (toYaml .) $root | fromYaml) -}} {{- end }} -{{- with .grpc -}} -{{- $overrideProbe = dict "grpc" . -}} +{{- with $values.grpc -}} +{{- $overrideProbe = dict "grpc" (tpl (toYaml .) $root | fromYaml) -}} {{- end -}} {{- $overrideProbe | toYaml -}} {{- end -}} @@ -258,8 +260,8 @@ template: {{- if .node.startupProbe.enabled }} {{- with .node.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -274,8 +276,8 @@ template: {{- if .node.readinessProbe.enabled }} {{- with .node.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 12 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 12 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -290,8 +292,8 @@ template: {{- if .node.livenessProbe.enabled }} {{- with .node.livenessProbe }} livenessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} diff --git a/charts/selenium-grid/templates/hub-deployment.yaml b/charts/selenium-grid/templates/hub-deployment.yaml index acb233e2c..77ffed0be 100644 --- a/charts/selenium-grid/templates/hub-deployment.yaml +++ b/charts/selenium-grid/templates/hub-deployment.yaml @@ -45,8 +45,8 @@ spec: {{- if .Values.hub.startupProbe.enabled }} {{- with .Values.hub.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -61,8 +61,8 @@ spec: {{- if .Values.hub.readinessProbe.enabled }} {{- with .Values.hub.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -77,8 +77,8 @@ spec: {{- if .Values.hub.livenessProbe.enabled }} {{- with .Values.hub.livenessProbe }} livenessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} diff --git a/charts/selenium-grid/templates/node-configmap.yaml b/charts/selenium-grid/templates/node-configmap.yaml index 3bddee26f..c6b1e6c99 100644 --- a/charts/selenium-grid/templates/node-configmap.yaml +++ b/charts/selenium-grid/templates/node-configmap.yaml @@ -12,9 +12,18 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: +{{- if .Values.isolateComponents }} + SE_DISTRIBUTOR_HOST: '{{ include "seleniumGrid.distributor.fullname" . }}.{{ .Release.Namespace }}' + SE_DISTRIBUTOR_PORT: '{{ .Values.components.distributor.port }}' + SE_ROUTER_HOST: '{{ include "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}' + SE_ROUTER_PORT: '{{ .Values.components.router.port }}' +{{- else }} + SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}' + SE_HUB_PORT: '{{ .Values.hub.port }}' +{{- end }} SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}' SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}' - SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" .}}' + SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}' {{- range $fileName, $value := .Values.nodeConfigMap.extraScripts }} {{- if not (empty $value) }} {{ $fileName | indent 2 -}}: | diff --git a/charts/selenium-grid/templates/router-deployment.yaml b/charts/selenium-grid/templates/router-deployment.yaml index 57c0c095f..088e29477 100644 --- a/charts/selenium-grid/templates/router-deployment.yaml +++ b/charts/selenium-grid/templates/router-deployment.yaml @@ -84,8 +84,8 @@ spec: {{- if .Values.components.router.startupProbe.enabled }} {{- with .Values.components.router.startupProbe }} startupProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -100,8 +100,8 @@ spec: {{- if .Values.components.router.readinessProbe.enabled }} {{- with .Values.components.router.readinessProbe }} readinessProbe: - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} @@ -116,8 +116,8 @@ spec: {{- if .Values.components.router.livenessProbe.enabled }} livenessProbe: {{- with .Values.components.router.livenessProbe }} - {{- if (ne (include "seleniumGrid.probe.fromUserDefine" .) "{}") }} - {{- include "seleniumGrid.probe.fromUserDefine" . | nindent 10 }} + {{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }} + {{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }} {{- else }} httpGet: scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index d023ceed4..fa62e0e0e 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -105,6 +105,7 @@ nodeConfigMap: extraScriptsDirectory: "/opt/selenium" extraScripts: nodePreStop.sh: + nodeProbe.sh: # Name of volume mount is used to mount scripts in the ConfigMap scriptVolumeMountName: # Custom annotations for configmap @@ -650,11 +651,13 @@ chromeNode: # Wait for pod startup startupProbe: enabled: true + exec: + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"] path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings @@ -810,11 +813,13 @@ firefoxNode: # Wait for pod startup startupProbe: enabled: true + exec: + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"] path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings @@ -968,11 +973,13 @@ edgeNode: # Wait for pod startup startupProbe: enabled: true + exec: + command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"] path: /status initialDelaySeconds: 0 - failureThreshold: 10 - timeoutSeconds: 5 periodSeconds: 5 + timeoutSeconds: 125 + failureThreshold: 25 successThreshold: 1 # Readiness probe settings diff --git a/tests/charts/ci/DeploymentAutoscaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml index 740e0465d..f249d96b2 100644 --- a/tests/charts/ci/DeploymentAutoscaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -1,5 +1,3 @@ -isolateComponents: false - autoscaling: enableWithExistingKEDA: true scalingType: deployment diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index 63b9d364f..cee4844e1 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -1,5 +1,3 @@ -isolateComponents: true - autoscaling: enableWithExistingKEDA: true scalingType: job diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml index 4ec462921..5679f5246 100644 --- a/tests/charts/ci/base-resources-values.yaml +++ b/tests/charts/ci/base-resources-values.yaml @@ -57,6 +57,12 @@ chromeNode: limits: cpu: 250m memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 firefoxNode: resources: @@ -66,6 +72,12 @@ firefoxNode: limits: cpu: 250m memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 edgeNode: resources: @@ -75,6 +87,12 @@ edgeNode: limits: cpu: 250m memory: 2500Mi + startupProbe: + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 400 + failureThreshold: 80 + successThreshold: 1 videoRecorder: resources: diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 3d3c76612..f16fd1160 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -27,6 +27,7 @@ CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} SE_ENABLE_TRACING=${SE_ENABLE_TRACING:-"false"} +SE_FULL_DISTRIBUTED_MODE=${SE_FULL_DISTRIBUTED_MODE:-"false"} cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then @@ -40,14 +41,17 @@ cleanup() { on_failure() { local exit_status=$? echo "Describe all resources in the ${SELENIUM_NAMESPACE} namespace for debugging purposes" - kubectl describe all -n ${SELENIUM_NAMESPACE} > tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt + kubectl describe all -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt + kubectl describe pod -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt echo "There is step failed with exit status $exit_status" cleanup exit $exit_status } # Trap ERR signal and call on_failure function -trap 'on_failure' ERR +trap 'on_failure' ERR EXIT + +touch tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt if [ -f .env ] then @@ -67,6 +71,7 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ --set autoscaling.scaledOptions.pollingInterval=${AUTOSCALING_POLL_INTERVAL} \ --set tracing.enabled=${SE_ENABLE_TRACING} \ +--set isolateComponents=${SE_FULL_DISTRIBUTED_MODE} \ " if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then @@ -122,7 +127,7 @@ export SELENIUM_GRID_TEST_HEADLESS=${SELENIUM_GRID_TEST_HEADLESS:-"false"} echo "Get pods status" kubectl get pods -n ${SELENIUM_NAMESPACE} -echo "Get all resources in the ${SELENIUM_NAMESPACE} namespace" -kubectl get all -n ${SELENIUM_NAMESPACE} > tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt +echo "Get all resources in all namespaces" +kubectl get all -A >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt cleanup