feat(otel_metrics): add OTEL metrics output support

Signed-off-by: Leonardo Di Giovanna <leonardodigiovanna1@gmail.com>

config.go

@@ -430,7 +430,10 @@ func getConfig() *types.Configuration {
 		Alertmanager:    types.AlertmanagerOutputConfig{ExtraLabels: make(map[string]string), ExtraAnnotations: make(map[string]string), CustomSeverityMap: make(map[types.PriorityType]string), CustomHeaders: make(map[string]string)},
 		CloudEvents:     types.CloudEventsOutputConfig{Extensions: make(map[string]string)},
 		GCP:             types.GcpOutputConfig{PubSub: types.GcpPubSub{CustomAttributes: make(map[string]string)}},
-		OTLP:            types.OTLPOutputConfig{Traces: types.OTLPTraces{ExtraEnvVars: make(map[string]string)}},
+		OTLP: types.OTLPOutputConfig{
+			Traces:  types.OTLPTraces{ExtraEnvVars: make(map[string]string)},
+			Metrics: types.OTLPMetrics{ExtraEnvVars: make(map[string]string)},
+		},
 	}
 
 	configFile := kingpin.Flag("config-file", "config file").Short('c').ExistingFile()
@@ -550,7 +553,7 @@ func getConfig() *types.Configuration {
 	v.SetDefault("OTLP.Traces.Endpoint", "")
 	v.SetDefault("OTLP.Traces.Protocol", "http/json")
 	// NOTE: we don't need to parse the OTLP.Traces.Headers field, as use it to
-	// set OTEL_EXPORTER_OTLP_TRACES_HEADERS (at otlp_init.go), which is then
+	// set OTEL_EXPORTER_OTLP_TRACES_HEADERS (at otlp_traces_init.go), which is then
 	// parsed by the OTLP SDK libs, see
 	// https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#otel_exporter_otlp_traces_headers
 	v.SetDefault("OTLP.Traces.Headers", "")
@@ -562,6 +565,15 @@ func getConfig() *types.Configuration {
 	// it to 1000ms by default, override-able via OTLP_DURATION environment variable.
 	v.SetDefault("OTLP.Traces.Duration", 1000)
 
+	v.SetDefault("OTLP.Metrics.Endpoint", "")
+	v.SetDefault("OTLP.Metrics.Protocol", "grpc")
+	// NOTE: we don't need to parse the OTLP.Metrics.Headers field, as use it to set OTEL_EXPORTER_OTLP_METRICS_HEADERS
+	// (at otlp_metrics.go), which is then parsed by the OTLP SDK libs.
+	v.SetDefault("OTLP.Metrics.Headers", "")
+	v.SetDefault("OTLP.Metrics.Timeout", 10000)
+	v.SetDefault("OTLP.Metrics.MinimumPriority", "")
+	v.SetDefault("OTLP.Metrics.CheckCert", true)
+
 	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	v.AutomaticEnv()
 	if *configFile != "" {
@@ -589,6 +601,7 @@ func getConfig() *types.Configuration {
 	v.GetStringMapString("AlertManager.CustomSeverityMap")
 	v.GetStringMapString("GCP.PubSub.CustomAttributes")
 	v.GetStringMapString("OTLP.Traces.ExtraEnvVars")
+	v.GetStringMapString("OTLP.Metrics.ExtraEnvVars")
 	if err := v.Unmarshal(c); err != nil {
 		log.Printf("[ERROR] : Error unmarshalling config : %s", err)
 	}
@@ -740,6 +753,21 @@ func getConfig() *types.Configuration {
 		}
 	}
 
+	if value, present := os.LookupEnv("OTLP_METRICS_EXTRAENVVARS"); present {
+		extraEnvVars := strings.Split(value, ",")
+		for _, extraEnvVarData := range extraEnvVars {
+			envName, envValue, found := strings.Cut(extraEnvVarData, ":")
+			envName, envValue = strings.TrimSpace(envName), strings.TrimSpace(envValue)
+			if !promKVNameRegex.MatchString(envName) {
+				log.Printf("[ERROR] : OTLPMetrics - Extra Env Var name '%v' is not valid", envName)
+			} else if found {
+				c.OTLP.Metrics.ExtraEnvVars[envName] = envValue
+			} else {
+				c.OTLP.Metrics.ExtraEnvVars[envName] = ""
+			}
+		}
+	}
+
 	if c.AWS.SecurityLake.Interval < 5 {
 		c.AWS.SecurityLake.Interval = 5
 	}
@@ -871,6 +899,7 @@ func getConfig() *types.Configuration {
 	c.OpenObserve.MinimumPriority = checkPriority(c.OpenObserve.MinimumPriority)
 	c.Dynatrace.MinimumPriority = checkPriority(c.Dynatrace.MinimumPriority)
 	c.SumoLogic.MinimumPriority = checkPriority(c.SumoLogic.MinimumPriority)
+	c.OTLP.Metrics.MinimumPriority = checkPriority(c.OTLP.Metrics.MinimumPriority)
 	c.Talon.MinimumPriority = checkPriority(c.Talon.MinimumPriority)
 
 	c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat)

config_example.yaml

@@ -543,6 +543,17 @@ otlp:
     # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
     # checkcert: true # Set if you want to skip TLS certificate validation (default: true)
 
+  metrics:
+    # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics
+    # protocol: "" # OTLP transport protocol to be used for metrics data; it can be "grpc" or "http/protobuf" (default: "grpc")
+    # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # extraenvvars: # Extra env vars (override the other settings) (default: "")
+      # OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # Minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default: "")
+    # checkcert: true # Set to false if you want to skip TLS certificate validation (only with https) (default: true)
+
 talon:
   # address: "" # Falco talon address, if not empty, Falco Talon output is enabled
   # checkcert: false # check if ssl certificate of the output is valid (default: true)

go.mod

@@ -33,9 +33,13 @@ require (
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18
 	go.opentelemetry.io/otel v1.30.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0
+	go.opentelemetry.io/otel/metric v1.30.0
 	go.opentelemetry.io/otel/sdk v1.30.0
+	go.opentelemetry.io/otel/sdk/metric v1.30.0
 	go.opentelemetry.io/otel/trace v1.30.0
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/sync v0.8.0
@@ -133,7 +137,6 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
-	go.opentelemetry.io/otel/metric v1.30.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect

go.sum

@@ -833,6 +833,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+n
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
 go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts=
 go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 h1:WypxHH02KX2poqqbaadmkMYalGyy/vil4HE4PM4nRJc=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0/go.mod h1:U79SV99vtvGSEBeeHnpgGJfTsnsdkWLpPN/CcHAzBSI=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 h1:JAv0Jwtl01UFiyWZEMiJZBiTlv5A50zNs8lsthXqIio=
@@ -841,6 +845,8 @@ go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4Q
 go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ=
 go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE=
 go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg=
+go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM=
+go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
 go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc=
 go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=

handlers.go

@@ -475,7 +475,13 @@ func forwardEvent(falcopayload types.FalcoPayload) {
 	}
 
 	if config.OTLP.Traces.Endpoint != "" && (falcopayload.Priority >= types.Priority(config.OTLP.Traces.MinimumPriority)) && (falcopayload.Source == syscall || falcopayload.Source == syscalls) {
-		go otlpClient.OTLPTracesPost(falcopayload)
+		go otlpTracesClient.OTLPTracesPost(falcopayload)
+	}
+
+	if config.OTLP.Metrics.Endpoint != "" &&
+		(falcopayload.Priority) >= types.Priority(config.OTLP.Metrics.MinimumPriority) &&
+		(falcopayload.Source == syscall || falcopayload.Source == syscalls) {
+		go otlpMetricsClient.OTLPMetricsPost(falcopayload)
 	}
 
 	if config.Talon.Address != "" && (falcopayload.Priority >= types.Priority(config.Talon.MinimumPriority) || falcopayload.Rule == testRule) {

main.go

@@ -78,7 +78,8 @@ var (
 	n8nClient           *outputs.Client
 	openObserveClient   *outputs.Client
 	dynatraceClient     *outputs.Client
-	otlpClient          *outputs.Client
+	otlpTracesClient    *outputs.Client
+	otlpMetricsClient   *outputs.Client
 	talonClient         *outputs.Client
 
 	statsdClient, dogstatsdClient *statsd.Client
@@ -801,12 +802,24 @@ func init() {
 
 	if config.OTLP.Traces.Endpoint != "" {
 		var err error
-		otlpClient, err = outputs.NewOtlpTracesClient(config, stats, promStats, statsdClient, dogstatsdClient)
+		otlpTracesClient, err = outputs.NewOtlpTracesClient(config, stats, promStats, statsdClient, dogstatsdClient)
 		if err != nil {
 			config.OTLP.Traces.Endpoint = ""
 		} else {
 			outputs.EnabledOutputs = append(outputs.EnabledOutputs, "OTLPTraces")
-			shutDownFuncs = append(shutDownFuncs, otlpClient.ShutDownFunc)
+			shutDownFuncs = append(shutDownFuncs, otlpTracesClient.ShutDownFunc)
+		}
+	}
+
+	if config.OTLP.Metrics.Endpoint != "" {
+		var err error
+		otlpMetricsClient, err = outputs.NewOTLPMetricsClient(context.Background(), config, stats, promStats,
+			statsdClient, dogstatsdClient)
+		if err != nil {
+			config.OTLP.Metrics.Endpoint = ""
+		} else {
+			outputs.EnabledOutputs = append(outputs.EnabledOutputs, "OTLPMetrics")
+			shutDownFuncs = append(shutDownFuncs, otlpMetricsClient.ShutDownFunc)
 		}
 	}