From e15390e2cf70f6669c155de011bf7ae9b705d5cd Mon Sep 17 00:00:00 2001 From: mackjmr Date: Mon, 30 Oct 2023 16:30:14 +0100 Subject: [PATCH 1/3] Add kafkaconsumerreceiver + dd kafka_consumer intg + deploy standalone jmx collector to staging --- ci/scripts/ci-deploy-staging.sh | 20 ++++++++++++++++++++ ci/values-jmx.yaml | 19 +++++++++++++++---- ci/values.yaml | 2 +- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ci/scripts/ci-deploy-staging.sh b/ci/scripts/ci-deploy-staging.sh index 5a1847c8bb29..ed12ce65499b 100644 --- a/ci/scripts/ci-deploy-staging.sh +++ b/ci/scripts/ci-deploy-staging.sh @@ -24,6 +24,26 @@ install_collector() { --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" helm list --all-namespaces + + helm -n ${namespace} uninstall opentelemetry-collector-deployment + + + if [ "$namespace" == "otel-staging" ]; then + install_deployment + fi +} + +install_deployment() { + release_name_deployment="opentelemetry-collector-deployment" + + # --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment + # as this would lead to duplicate metrics. + helm --debug upgrade "${release_name_deployment}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \ + -f ./ci/values-jmx.yaml \ + --set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \ + --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" \ + --set nodeSelector.alpha\\.eksctl\\.io/nodegroup-name=ng-3 + } ########################################################################################################### diff --git a/ci/values-jmx.yaml b/ci/values-jmx.yaml index 4b93eb5721d4..5ad6b85f5cf0 100644 --- a/ci/values-jmx.yaml +++ b/ci/values-jmx.yaml @@ -15,7 +15,7 @@ extraEnvs: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: http://$(OTEL_COLLECTOR_NAME):4317 - name: OTEL_RESOURCE_ATTRIBUTES - value: kafka_source=jmxreceiver + value: kafka_source=deploymentcol config: receivers: jmx: @@ -42,12 +42,23 @@ config: endpoint: "opentelemetry-demo-zookeeper:2181" collection_interval: 20s initial_delay: 1s + kafkametrics: + brokers: "opentelemetry-demo-kafka:9092" + protocol_version: 2.0.0 + scrapers: + - brokers + - topics + - consumers exporters: + logging: + verbosity: detailed otlp: endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT} tls: insecure: true processors: + resourcedetection/env: + detectors: [env] batch: send_batch_max_size: 1000 send_batch_size: 100 @@ -55,6 +66,6 @@ config: service: pipelines: metrics: - receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper] - processors: [batch] - exporters: [otlp] \ No newline at end of file + receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper, kafkametrics] + processors: [batch, resourcedetection/env] + exporters: [otlp, logging] \ No newline at end of file diff --git a/ci/values.yaml b/ci/values.yaml index cdd7db70e0f3..885350246b70 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -185,7 +185,7 @@ config: metrics: receivers: [otlp, hostmetrics, prometheus] processors: [resourcedetection, k8sattributes, batch] - exporters: [datadog] + exporters: [datadog, logging] traces: receivers: [otlp] processors: [resourcedetection, k8sattributes, batch] From f0b352848f0b6f6372ec7f619b38c51de2729d8d Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 31 Oct 2023 17:50:40 +0100 Subject: [PATCH 2/3] remove uninstall --- ci/scripts/ci-deploy-staging.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/scripts/ci-deploy-staging.sh b/ci/scripts/ci-deploy-staging.sh index ed12ce65499b..36749fd85d7c 100644 --- a/ci/scripts/ci-deploy-staging.sh +++ b/ci/scripts/ci-deploy-staging.sh @@ -25,9 +25,6 @@ install_collector() { --set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" helm list --all-namespaces - helm -n ${namespace} uninstall opentelemetry-collector-deployment - - if [ "$namespace" == "otel-staging" ]; then install_deployment fi From 647c023fafab2d19574d2b4594f8995682cc4970 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Thu, 2 Nov 2023 13:46:05 +0100 Subject: [PATCH 3/3] remove logging exporter from pipeline --- ci/values-jmx.yaml | 2 +- ci/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/values-jmx.yaml b/ci/values-jmx.yaml index 5ad6b85f5cf0..b44b566924e5 100644 --- a/ci/values-jmx.yaml +++ b/ci/values-jmx.yaml @@ -68,4 +68,4 @@ config: metrics: receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper, kafkametrics] processors: [batch, resourcedetection/env] - exporters: [otlp, logging] \ No newline at end of file + exporters: [otlp] \ No newline at end of file diff --git a/ci/values.yaml b/ci/values.yaml index 885350246b70..cdd7db70e0f3 100644 --- a/ci/values.yaml +++ b/ci/values.yaml @@ -185,7 +185,7 @@ config: metrics: receivers: [otlp, hostmetrics, prometheus] processors: [resourcedetection, k8sattributes, batch] - exporters: [datadog, logging] + exporters: [datadog] traces: receivers: [otlp] processors: [resourcedetection, k8sattributes, batch]