From cd10250efd94772b596299600a583e56598f82ab Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Sun, 18 Feb 2024 17:06:38 +0700 Subject: [PATCH] feat: enable tracing observability in docker-compose and helm chart (#2137) Signed-off-by: Viet Nguyen Duc --- Base/Dockerfile | 3 ++ Distributor/Dockerfile | 2 + .../start-selenium-grid-distributor.sh | 18 ++++++- EventBus/Dockerfile | 2 + EventBus/start-selenium-grid-eventbus.sh | 18 ++++++- Hub/Dockerfile | 2 + Hub/start-selenium-grid-hub.sh | 18 ++++++- Makefile | 4 +- NodeBase/Dockerfile | 1 + NodeBase/start-selenium-node.sh | 18 ++++++- NodeChrome/Dockerfile | 2 + NodeDocker/Dockerfile | 1 + NodeDocker/start-selenium-grid-docker.sh | 18 ++++++- NodeEdge/Dockerfile | 2 + NodeFirefox/Dockerfile | 2 + README.md | 13 +++-- Router/Dockerfile | 2 + Router/start-selenium-grid-router.sh | 18 ++++++- SessionQueue/Dockerfile | 2 + .../start-selenium-grid-session-queue.sh | 18 ++++++- Sessions/Dockerfile | 2 + Sessions/start-selenium-grid-sessions.sh | 18 ++++++- Standalone/Dockerfile | 2 +- Standalone/start-selenium-standalone.sh | 18 ++++++- StandaloneDocker/Dockerfile | 2 + .../start-selenium-grid-docker.sh | 18 ++++++- charts/selenium-grid/Chart.yaml | 4 ++ charts/selenium-grid/README.md | 27 ++++++++++ charts/selenium-grid/templates/_helpers.tpl | 9 ++++ .../templates/distributor-deployment.yaml | 2 + .../templates/event-bus-deployment.yaml | 2 + .../templates/hub-deployment.yaml | 2 + .../templates/jaeger-ingress.yaml | 51 +++++++++++++++++++ .../templates/logging-configmap.yaml | 6 +++ .../templates/router-deployment.yaml | 2 + .../templates/session-map-deployment.yaml | 2 + .../templates/session-queuer-deployment.yaml | 2 + charts/selenium-grid/values.yaml | 36 +++++++++++++ docker-compose-v2-tracing.yml | 18 ++++--- docker-compose-v3-full-grid-tracing.yml | 35 ++++++++----- docker-compose-v3-tracing.yml | 19 ++++--- tests/charts/ci/base-auth-ingress-values.yaml | 3 +- tests/charts/ci/base-tracing-values.yaml | 8 --- tests/charts/config/ct.yaml | 1 + tests/charts/make/chart_cluster_setup.sh | 5 +- tests/charts/make/chart_test.sh | 2 + tests/charts/refValues/sample-aws.yaml | 3 +- tests/charts/refValues/simplex-minikube.yaml | 3 +- 48 files changed, 406 insertions(+), 60 deletions(-) create mode 100644 charts/selenium-grid/templates/jaeger-ingress.yaml diff --git a/Base/Dockerfile b/Base/Dockerfile index 27a9980a3..91a216c6a 100644 --- a/Base/Dockerfile +++ b/Base/Dockerfile @@ -140,6 +140,9 @@ ENV SE_BIND_HOST false # Boolean value, maps "--reject-unsupported-caps" ENV SE_REJECT_UNSUPPORTED_CAPS false +ENV SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED true +ENV SE_OTEL_TRACES_EXPORTER "otlp" + # A too high maximum number of file descriptors (with the default value # inherited from the docker host) can cause issues with some of our tools: # - sanitizers hanging: https://github.com/google/sanitizers/issues/1662 diff --git a/Distributor/Dockerfile b/Distributor/Dockerfile index 68dd3a303..7ddae88b6 100644 --- a/Distributor/Dockerfile +++ b/Distributor/Dockerfile @@ -21,3 +21,5 @@ COPY selenium-grid-distributor.conf /etc/supervisor/conf.d/ ENV SE_SESSION_REQUEST_TIMEOUT 300 # In seconds, maps to "--session-retry-interval" ENV SE_SESSION_RETRY_INTERVAL 15 + +ENV SE_OTEL_SERVICE_NAME "selenium-distributor" diff --git a/Distributor/start-selenium-grid-distributor.sh b/Distributor/start-selenium-grid-distributor.sh index e4e58215c..f8c2c0807 100755 --- a/Distributor/start-selenium-grid-distributor.sh +++ b/Distributor/start-selenium-grid-distributor.sh @@ -95,12 +95,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( /opt/selenium/browser_name RUN google-chrome --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"goog:chromeOptions\": {\"binary\": \"/usr/bin/google-chrome\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-chrome" diff --git a/NodeDocker/Dockerfile b/NodeDocker/Dockerfile index b65820cec..495dc9c78 100644 --- a/NodeDocker/Dockerfile +++ b/NodeDocker/Dockerfile @@ -28,3 +28,4 @@ COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-docker.sh \ COPY selenium-grid-docker.conf /etc/supervisor/conf.d/ +ENV SE_OTEL_SERVICE_NAME "selenium-node-docker" diff --git a/NodeDocker/start-selenium-grid-docker.sh b/NodeDocker/start-selenium-grid-docker.sh index 935fbee46..1396d0e48 100755 --- a/NodeDocker/start-selenium-grid-docker.sh +++ b/NodeDocker/start-selenium-grid-docker.sh @@ -60,12 +60,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$( /opt/selenium/browser_name RUN microsoft-edge --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"ms:edgeOptions\": {\"binary\": \"/usr/bin/microsoft-edge\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-edge" diff --git a/NodeFirefox/Dockerfile b/NodeFirefox/Dockerfile index 251be9b29..92d2684c5 100644 --- a/NodeFirefox/Dockerfile +++ b/NodeFirefox/Dockerfile @@ -44,3 +44,5 @@ USER ${SEL_UID} RUN echo "firefox" > /opt/selenium/browser_name RUN firefox --version | awk '{print $3}' > /opt/selenium/browser_version RUN echo "\"moz:firefoxOptions\": {\"binary\": \"/usr/bin/firefox\"}" > /opt/selenium/browser_binary_location + +ENV SE_OTEL_SERVICE_NAME "selenium-node-firefox" diff --git a/README.md b/README.md index fdcdc0f08..a726755bc 100644 --- a/README.md +++ b/README.md @@ -1267,26 +1267,29 @@ In order to enable tracing in the Selenium Grid container, the following command ```bash docker network create grid -docker run -d -p 16686:16686 -p 14250:14250 --net grid --name jaeger jaegertracing/all-in-one:1.17 +docker run -d -p 16686:16686 -p 14250:14250 -p 4317:4317 --net grid --name jaeger jaegertracing/all-in-one:1.54 docker run -d -p 4442-4444:4442-4444 --net grid --name selenium-hub selenium/hub:4.17.0-20240123 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ selenium/node-chrome:4.17.0-20240123 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ selenium/node-edge:4.17.0-20240123 docker run -d --net grid -e SE_EVENT_BUS_HOST=selenium-hub \ --shm-size="2g" \ -e SE_ENABLE_TRACING=true \ - -e JAVA_OPTS="-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox" \ + -e SE_OTEL_TRACES_EXPORTER=otlp \ + -e SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 \ -e SE_EVENT_BUS_PUBLISH_PORT=4442 \ -e SE_EVENT_BUS_SUBSCRIBE_PORT=4443 \ selenium/node-firefox:4.17.0-20240123 @@ -1298,7 +1301,7 @@ You can also refer to the below docker-compose yaml files to be able to start a * Simple Grid [v2 yaml file](docker-compose-v2-tracing.yml) * Dynamic Grid [v3 yaml file](docker-compose-v3-full-grid-tracing.yml) -You can view the [Jaegar UI](http://localhost:16686/) and trace your request. +You can view the [Jaeger UI](http://localhost:16686/) and trace your request. ___ ## Troubleshooting diff --git a/Router/Dockerfile b/Router/Dockerfile index 5a495b22e..cf1d7029b 100644 --- a/Router/Dockerfile +++ b/Router/Dockerfile @@ -21,3 +21,5 @@ COPY --chown="${SEL_UID}:${SEL_GID}" start-selenium-grid-router.sh \ /opt/bin/ COPY selenium-grid-router.conf /etc/supervisor/conf.d/ + +ENV SE_OTEL_SERVICE_NAME "selenium-router" diff --git a/Router/start-selenium-grid-router.sh b/Router/start-selenium-grid-router.sh index 3288abb15..d1ef46666 100755 --- a/Router/start-selenium-grid-router.sh +++ b/Router/start-selenium-grid-router.sh @@ -95,12 +95,28 @@ fi EXTRA_LIBS="" -if [ ! -z "$SE_ENABLE_TRACING" ]; then +if [ "$SE_ENABLE_TRACING" = "true" ]; then EXTERNAL_JARS=$(=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }}-jaeger-ingress + namespace: {{ .Release.Namespace }} + labels: + {{- include "seleniumGrid.commonLabels" . | nindent 4 }} + {{- with .Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.tracing.ingress.annotations }} + annotations: + {{- . | toYaml | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if and (or .Values.tls.enabled .Values.tls.ingress.generateTLS) (tpl .Values.ingress.hostname $) (not .Values.ingress.tls) }} + tls: + - hosts: + - {{ tpl .Values.ingress.hostname $ | quote }} + secretName: {{ include "seleniumGrid.tls.fullname" . | quote }} + {{- else if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ tpl . $ | quote }} + {{- end }} + secretName: {{ tpl (.secretName) $ | quote }} + {{- end }} + {{- end }} + rules: + {{- if $.Values.ingress.hostname }} + - host: {{ tpl $.Values.ingress.hostname $ }} + http: + {{- else }} + - http: + {{- end }} + paths: + {{- with .Values.tracing.ingress.paths }} + {{- tpl (toYaml . | nindent 10) $ }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/logging-configmap.yaml b/charts/selenium-grid/templates/logging-configmap.yaml index d8b5ab978..12ea1570d 100644 --- a/charts/selenium-grid/templates/logging-configmap.yaml +++ b/charts/selenium-grid/templates/logging-configmap.yaml @@ -13,3 +13,9 @@ metadata: {{- end }} data: SE_LOG_LEVEL: "{{ default "INFO" .Values.global.seleniumGrid.logLevel }}" +{{- if (eq (include "seleniumGrid.enableTracing" .) "true") }} + SE_ENABLE_TRACING: "true" + SE_OTEL_TRACES_EXPORTER: {{ .Values.tracing.exporter | quote }} + SE_OTEL_JAVA_GLOBAL_AUTOCONFIGURE_ENABLED: {{ .Values.tracing.globalAutoConfigure | quote }} + SE_OTEL_EXPORTER_ENDPOINT: {{ tpl .Values.tracing.exporterEndpoint $ | quote }} +{{- end }} diff --git a/charts/selenium-grid/templates/router-deployment.yaml b/charts/selenium-grid/templates/router-deployment.yaml index fa5d2d50a..57c0c095f 100644 --- a/charts/selenium-grid/templates/router-deployment.yaml +++ b/charts/selenium-grid/templates/router-deployment.yaml @@ -33,6 +33,8 @@ spec: image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.router.imageName $imageTag }} imagePullPolicy: {{ .Values.components.router.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.router.fullname" . }}' - name: SE_ROUTER_HOST value: '{{ template "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}' - name: SE_ROUTER_PORT diff --git a/charts/selenium-grid/templates/session-map-deployment.yaml b/charts/selenium-grid/templates/session-map-deployment.yaml index bfc788d7f..1057d00cc 100644 --- a/charts/selenium-grid/templates/session-map-deployment.yaml +++ b/charts/selenium-grid/templates/session-map-deployment.yaml @@ -33,6 +33,8 @@ spec: image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.sessionMap.imageName $imageTag }} imagePullPolicy: {{ .Values.components.sessionMap.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.sessionMap.fullname" . }}' - name: SE_SESSIONS_HOST value: '{{ template "seleniumGrid.sessionMap.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSIONS_PORT diff --git a/charts/selenium-grid/templates/session-queuer-deployment.yaml b/charts/selenium-grid/templates/session-queuer-deployment.yaml index 3298f4182..9765a315a 100644 --- a/charts/selenium-grid/templates/session-queuer-deployment.yaml +++ b/charts/selenium-grid/templates/session-queuer-deployment.yaml @@ -33,6 +33,8 @@ spec: image: {{ printf "%s/%s:%s" $imageRegistry .Values.components.sessionQueue.imageName $imageTag }} imagePullPolicy: {{ .Values.components.sessionQueue.imagePullPolicy }} env: + - name: SE_OTEL_SERVICE_NAME + value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}' - name: SE_SESSION_QUEUE_HOST value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}.{{ .Release.Namespace }}' - name: SE_SESSION_QUEUE_PORT diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index c982713fd..d023ceed4 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -492,6 +492,24 @@ hub: # Priority class name for selenium-hub pods priorityClassName: "" +tracing: + enabled: false + enabledWithExistingEndpoint: false + exporter: otlp + exporterEndpoint: 'http://{{ .Release.Name }}-jaeger-collector.{{ .Release.Namespace }}:4317' + globalAutoConfigure: true + ingress: + enabled: true + annotations: + paths: + - backend: + service: + name: '{{ .Release.Name }}-jaeger-query' + port: + number: 16686 + path: &jaegerBasePath "/jaeger" + pathType: Prefix + # Keda scaled object configuration autoscaling: # Enable autoscaling. Implies installing KEDA @@ -1146,3 +1164,21 @@ ingress-nginx: controller: admissionWebhooks: enabled: false + +# Configuration for dependency chart jaeger +jaeger: + provisionDataStore: + cassandra: false + allInOne: + enabled: true + extraEnv: + - name: QUERY_BASE_PATH + value: *jaegerBasePath + storage: + type: none + agent: + enabled: false + collector: + enabled: false + query: + enabled: false diff --git a/docker-compose-v2-tracing.yml b/docker-compose-v2-tracing.yml index 5cdb207b6..342c4f380 100644 --- a/docker-compose-v2-tracing.yml +++ b/docker-compose-v2-tracing.yml @@ -3,8 +3,8 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v2-tracing.yml down` version: '2' services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - "14250:14250" @@ -18,7 +18,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 ports: - "6900:5900" @@ -32,7 +33,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 ports: - "6901:5900" @@ -46,7 +48,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 ports: - "6902:5900" @@ -57,7 +60,8 @@ services: - "4443:4443" - "4444:4444" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 diff --git a/docker-compose-v3-full-grid-tracing.yml b/docker-compose-v3-full-grid-tracing.yml index 5ddfd0cee..539842a6f 100644 --- a/docker-compose-v3-full-grid-tracing.yml +++ b/docker-compose-v3-full-grid-tracing.yml @@ -3,11 +3,12 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-full-grid-tracing.yml down` version: "3" services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - "14250:14250" + - "4317:4317" selenium-event-bus: image: selenium/event-bus:4.17.0-20240123 container_name: selenium-event-bus @@ -16,10 +17,11 @@ services: - "4443:4443" - "5557:5557" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-event-bus + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 selenium-sessions: image: selenium/sessions:4.17.0-20240123 container_name: selenium-sessions @@ -32,7 +34,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-sessions + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 selenium-session-queue: image: selenium/session-queue:4.17.0-20240123 container_name: selenium-session-queue @@ -40,7 +43,8 @@ services: - "5559:5559" environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-session-queue + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 selenium-distributor: image: selenium/distributor:4.17.0-20240123 container_name: selenium-distributor @@ -59,7 +63,8 @@ services: - SE_SESSION_QUEUE_HOST=selenium-session-queue - SE_SESSION_QUEUE_PORT=5559 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-distributor + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 selenium-router: image: selenium/router:4.17.0-20240123 container_name: selenium-router @@ -76,8 +81,9 @@ services: - SE_SESSIONS_MAP_PORT=5556 - SE_SESSION_QUEUE_HOST=selenium-session-queue - SE_SESSION_QUEUE_PORT=5559 - - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-router + - SE_ENABLE_TRACING=true + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 chrome: image: selenium/node-chrome:4.17.0-20240123 shm_size: 2gb @@ -88,7 +94,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 edge: image: selenium/node-edge:4.17.0-20240123 shm_size: 2gb @@ -98,8 +105,9 @@ services: - SE_EVENT_BUS_HOST=selenium-event-bus - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_ENABLE_TRACING=true + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 firefox: image: selenium/node-firefox:4.17.0-20240123 shm_size: 2gb @@ -110,4 +118,5 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox \ No newline at end of file + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 diff --git a/docker-compose-v3-tracing.yml b/docker-compose-v3-tracing.yml index 7b099880f..5bb676504 100644 --- a/docker-compose-v3-tracing.yml +++ b/docker-compose-v3-tracing.yml @@ -3,11 +3,12 @@ # To stop the execution, hit Ctrl+C, and then `docker-compose -f docker-compose-v3-tracing.yml down` version: "3" services: - jaegar: - image: jaegertracing/all-in-one:1.17 + jaeger: + image: jaegertracing/all-in-one:1.54 ports: - "16686:16686" - "14250:14250" + - "4317:4317" chrome: image: selenium/node-chrome:4.17.0-20240123 shm_size: 2gb @@ -18,7 +19,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-chrome + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 edge: image: selenium/node-edge:4.17.0-20240123 @@ -30,7 +32,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-edge + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 firefox: image: selenium/node-firefox:4.17.0-20240123 @@ -42,7 +45,8 @@ services: - SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-node-firefox + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 selenium-hub: image: selenium/hub:4.17.0-20240123 @@ -52,7 +56,8 @@ services: - "4443:4443" - "4444:4444" depends_on: - - jaegar + - jaeger environment: - SE_ENABLE_TRACING=true - - JAVA_OPTS=-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://jaegar:14250 -Dotel.resource.attributes=service.name=selenium-hub + - SE_OTEL_TRACES_EXPORTER=otlp #or jaeger + - SE_OTEL_EXPORTER_ENDPOINT=http://jaeger:4317 #or http://jaeger:14250 diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml index 0d069fa56..5efe3c104 100644 --- a/tests/charts/ci/base-auth-ingress-values.yaml +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -5,11 +5,10 @@ global: ingress: annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - ingressClassName: nginx + className: nginx hostname: "" paths: - path: /selenium(/|$)(.*) diff --git a/tests/charts/ci/base-tracing-values.yaml b/tests/charts/ci/base-tracing-values.yaml index 6520f52ce..cbbf57a76 100644 --- a/tests/charts/ci/base-tracing-values.yaml +++ b/tests/charts/ci/base-tracing-values.yaml @@ -6,10 +6,6 @@ hub: value: "5" - name: SE_REJECT_UNSUPPORTED_CAPS value: "false" - - name: SE_ENABLE_TRACING - value: "true" - - name: SE_JAVA_OPTS - value: "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" components: extraEnvironmentVariables: @@ -19,7 +15,3 @@ components: value: "5" - name: SE_REJECT_UNSUPPORTED_CAPS value: "false" - - name: SE_ENABLE_TRACING - value: "true" - - name: SE_JAVA_OPTS - value: "-Dotel.traces.exporter=jaeger -Dotel.exporter.jaeger.endpoint=http://{{ .Release.Name }}-jaeger-all-in-one-headless:14250 -Dotel.resource.attributes=service.name=selenium -Dotel.java.global-autoconfigure.enabled=true" diff --git a/tests/charts/config/ct.yaml b/tests/charts/config/ct.yaml index 2b2446823..bad585538 100755 --- a/tests/charts/config/ct.yaml +++ b/tests/charts/config/ct.yaml @@ -6,6 +6,7 @@ chart-dirs: chart-repos: - kedacore=https://kedacore.github.io/charts - ingressNginx=https://kubernetes.github.io/ingress-nginx + - jaegertracing=https://jaegertracing.github.io/helm-charts upgrade: false helm-extra-args: --timeout 600s check-version-increment: false diff --git a/tests/charts/make/chart_cluster_setup.sh b/tests/charts/make/chart_cluster_setup.sh index c34bd522c..35ff2da4b 100755 --- a/tests/charts/make/chart_cluster_setup.sh +++ b/tests/charts/make/chart_cluster_setup.sh @@ -15,6 +15,9 @@ SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"} SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"} WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"} SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the cluster after the test run +KUBERNETES_VERSION=${KUBERNETES_VERSION:-$(curl -L -s https://dl.k8s.io/release/stable.txt)} +CNI=${CNI:-"calico"} # auto, calico, cilium +CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"docker"} # docker, containerd, cri-o # Function to clean up for retry step on workflow cleanup() { @@ -53,7 +56,7 @@ elif [ "${CLUSTER}" = "minikube" ]; then sudo chmod 777 /tmp export CHANGE_MINIKUBE_NONE_USER=true sudo -SE minikube start --vm-driver=none --cpus ${CPUs} --memory ${MEMORY} \ - --kubernetes-version=$(curl -L -s https://dl.k8s.io/release/stable.txt) --network-plugin=cni --cni=calico + --kubernetes-version=${KUBERNETES_VERSION} --network-plugin=cni --cni=${CNI} --container-runtime=${CONTAINER_RUNTIME} sudo chown -R $USER $HOME/.kube $HOME/.minikube fi diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index d18f5f90d..3d3c76612 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -26,6 +26,7 @@ SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the clust CHART_CERT_PATH=${CHART_CERT_PATH:-"${CHART_PATH}/certs/selenium.pem"} SSL_CERT_DIR=${SSL_CERT_DIR:-"/etc/ssl/certs"} VIDEO_TAG=${VIDEO_TAG:-"latest"} +SE_ENABLE_TRACING=${SE_ENABLE_TRACING:-"false"} cleanup() { if [ "${SKIP_CLEANUP}" = "false" ]; then @@ -65,6 +66,7 @@ HELM_COMMAND_SET_IMAGES=" \ --set global.seleniumGrid.nodesImageTag=${VERSION} \ --set global.seleniumGrid.videoImageTag=${VIDEO_TAG} \ --set autoscaling.scaledOptions.pollingInterval=${AUTOSCALING_POLL_INTERVAL} \ +--set tracing.enabled=${SE_ENABLE_TRACING} \ " if [ "${SELENIUM_GRID_AUTOSCALING}" = "true" ]; then diff --git a/tests/charts/refValues/sample-aws.yaml b/tests/charts/refValues/sample-aws.yaml index 2112f34de..dc7b7e61d 100644 --- a/tests/charts/refValues/sample-aws.yaml +++ b/tests/charts/refValues/sample-aws.yaml @@ -11,11 +11,10 @@ global: ingress: enabled: true annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - ingressClassName: nginx + className: nginx hostname: "aws.ndviet.org" # Replace with your hostname paths: - path: /selenium(/|$)(.*) diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 2708aa1cd..9fc5162e8 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -22,11 +22,10 @@ tls: ingress: enabled: true annotations: - kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" - ingressClassName: nginx + className: nginx hostname: "" paths: - path: /selenium(/|$)(.*)