Add kafkaconsumerreceiver + dd kafka_consumer intg + deploy standalon…

…e jmx collector to staging

ci/scripts/ci-deploy-staging.sh

@@ -24,6 +24,26 @@ install_collector() {
 		--set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \
 		--set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib"
 	helm list --all-namespaces
+
+	helm -n ${namespace} uninstall opentelemetry-collector-deployment
+
+
+	if [ "$namespace" == "otel-staging" ]; then
+		install_deployment
+	fi
+}
+
+install_deployment() {
+	release_name_deployment="opentelemetry-collector-deployment"
+
+	# --install collector that fetches jmx metrics. The jmx receiver cannot be used in the daemonset deployment
+	# as this would lead to duplicate metrics.
+	helm --debug upgrade "${release_name_deployment}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \
+		-f ./ci/values-jmx.yaml \
+		--set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \
+		--set-string image.repository="601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib" \
+		--set nodeSelector.alpha\\.eksctl\\.io/nodegroup-name=ng-3
+
 }
 
 ###########################################################################################################

ci/values-jmx.yaml

@@ -15,7 +15,7 @@ extraEnvs:
   - name: OTEL_EXPORTER_OTLP_ENDPOINT
     value: http://$(OTEL_COLLECTOR_NAME):4317
   - name: OTEL_RESOURCE_ATTRIBUTES
-    value: kafka_source=jmxreceiver
+    value: kafka_source=deploymentcol
 config:
   receivers:
     jmx:
@@ -42,19 +42,30 @@ config:
       endpoint: "opentelemetry-demo-zookeeper:2181"
       collection_interval: 20s
       initial_delay: 1s
+    kafkametrics:
+      brokers: "opentelemetry-demo-kafka:9092"
+      protocol_version: 2.0.0
+      scrapers:
+        - brokers
+        - topics
+        - consumers
   exporters:
+    logging:
+      verbosity: detailed
     otlp:
       endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT}
       tls:
         insecure: true
   processors:
+    resourcedetection/env:
+      detectors: [env]
     batch:
       send_batch_max_size: 1000
       send_batch_size: 100
       timeout: 10s
   service:
     pipelines:
       metrics:
-        receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper]
-        processors: [batch]
-        exporters: [otlp]
+        receivers: [jmx, jmx/jvm, jmx/consumer, jmx/producer, zookeeper, kafkametrics]
+        processors: [batch, resourcedetection/env]
+        exporters: [otlp, logging]

ci/values.yaml

@@ -185,7 +185,7 @@ config:
       metrics:
         receivers: [otlp, hostmetrics, prometheus]
         processors: [resourcedetection, k8sattributes, batch]
-        exporters: [datadog]
+        exporters: [datadog, logging]
       traces:
         receivers: [otlp]
         processors: [resourcedetection, k8sattributes, batch]