[WIP] Parse and use ES Beats config as OTel ES exporter

go.mod

@@ -226,9 +226,7 @@ require (
 	go.elastic.co/apm/v2 v2.6.0
 	go.mongodb.org/mongo-driver v1.5.1
 	go.opentelemetry.io/collector/component v0.110.0
-	go.opentelemetry.io/collector/config/confighttp v0.110.0
 	go.opentelemetry.io/collector/config/configopaque v1.16.0
-	go.opentelemetry.io/collector/config/configtls v1.16.0
 	go.opentelemetry.io/collector/confmap v1.16.0
 	go.opentelemetry.io/collector/confmap/provider/fileprovider v1.16.0
 	go.opentelemetry.io/collector/confmap/provider/httpprovider v0.110.0
@@ -294,6 +292,7 @@ require (
 	github.com/containerd/containerd v1.7.20 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 // indirect
+	github.com/creack/pty v1.1.20 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
@@ -371,7 +370,7 @@ require (
 	github.com/markbates/pkger v0.17.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -395,6 +394,7 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/prometheus/client_golang v1.20.4 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
@@ -419,8 +419,10 @@ require (
 	go.opentelemetry.io/collector/component/componentstatus v0.110.0 // indirect
 	go.opentelemetry.io/collector/config/configauth v0.110.0 // indirect
 	go.opentelemetry.io/collector/config/configcompression v1.16.0 // indirect
+	go.opentelemetry.io/collector/config/confighttp v0.110.0 // indirect
 	go.opentelemetry.io/collector/config/configretry v1.16.0 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.110.0 // indirect
+	go.opentelemetry.io/collector/config/configtls v1.16.0 // indirect
 	go.opentelemetry.io/collector/config/internal v0.110.0 // indirect
 	go.opentelemetry.io/collector/connector v0.110.0 // indirect
 	go.opentelemetry.io/collector/connector/connectorprofiles v0.110.0 // indirect
@@ -506,9 +508,3 @@ replace (
 	github.com/meraki/dashboard-api-go/v3 => github.com/tommyers-elastic/dashboard-api-go/v3 v3.0.0-20240913150833-a945473a8f25
 	github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19
 )
-
-// replace go.opentelemetry.io/collector/otelcol => /Users/vihasmakwana/Desktop/Vihas/OTeL/opentelemetry-collector/otelcol
-
-// replace go.opentelemetry.io/collector/confmap => /Users/vihasmakwana/Desktop/Vihas/OTeL/opentelemetry-collector/confmap
-
-// replace go.opentelemetry.io/collector/confmap/provider/fileprovider => /Users/vihasmakwana/Desktop/Vihas/OTeL/opentelemetry-collector/confmap/provider/fileprovider

go.sum

@@ -479,8 +479,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
-github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4=
+github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
@@ -1326,8 +1326,9 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
@@ -1582,6 +1583,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

libbeat/outputs/elasticsearch/config_otel.go

@@ -84,7 +84,8 @@ func ToOTelConfig(beatCfg *config.C) (map[string]any, error) {
 	}
 
 	otelYAMLCfg := map[string]any{
-		"logs_index":  esToOTelOptions.Index,    // index
+		"logs_index":  esToOTelOptions.Index, // index
+		"index":       esToOTelOptions.Index,
 		"pipeline":    esToOTelOptions.Pipeline, // pipeline
 		"endpoints":   hosts,                    // hosts, protocol, path, port
 		"num_workers": workersCfg.NumWorkers(),  // worker/workers
@@ -113,6 +114,11 @@ func ToOTelConfig(beatCfg *config.C) (map[string]any, error) {
 			"enabled":        true,
 			"max_size_items": escfg.BulkMaxSize, // bulk_max_size
 		},
+
+		// TODO (Tiago): Trying to make things work, remove later
+		"mapping": map[string]any{
+			"mode": "ecs",
+		},
 	}
 
 	return otelYAMLCfg, nil

x-pack/filebeat/cmd/customProvider/beatprovider.go

@@ -38,7 +38,6 @@ func (fmp *provider) Retrieve(_ context.Context, uri string, _ confmap.WatcherFu
 	cfg, err := cfgfile.Load(filepath.Clean(uri[len(schemeName)+1:]), nil)
 	if err != nil {
 		return nil, err
-
 	}
 
 	esCfg, err := elasticsearch.ToOTelConfig(cfg)
@@ -51,8 +50,12 @@ func (fmp *provider) Retrieve(_ context.Context, uri string, _ confmap.WatcherFu
 
 	cfg.SetChild("output", -1, newCfg)
 
+	fbCfg, err := cfg.Child("receivers.filebeatreceiver", -1)
+	if err != nil {
+		return nil, fmt.Errorf("cannot extract Filebeat config from  receivers: %w", err)
+	}
 	var receiverMap map[string]any
-	cfg.Unpack(&receiverMap)
+	fbCfg.Unpack(&receiverMap)
 
 	cfgMap := map[string]any{
 		"exporters": map[string]any{
@@ -63,10 +66,11 @@ func (fmp *provider) Retrieve(_ context.Context, uri string, _ confmap.WatcherFu
 			"filebeatreceiver": receiverMap,
 		},
 		"service": map[string]any{
-			"pipeline": map[string]any{
+			"pipelines": map[string]any{
 				"logs": map[string]any{
 					"exporters": []string{
 						"debug",
+						"elasticsearch",
 					},
 					"receivers": []string{"filebeatreceiver"},
 				},

x-pack/filebeat/cmd/otel.yml

@@ -15,9 +15,9 @@ receivers:
                       - /var/lib/docker/containers/*/*.log
       inputs:
         - type: filestream
+          id: my-filestream
           enabled: true
-          paths: /home/ubuntu/hello.log
-          take_over: false
+          paths: /tmp/flog.log
     output:
       otelconsumer:
     logging:
@@ -29,17 +29,25 @@ receivers:
           target: project
           fields:
             name: myproject
-
-
 exporters:
   debug:
   elasticsearch:
     user: elastic
-    password: ZA1kgoQPpp3bW5jzXi5CC0eT
-    endpoint: https://my-deployment-3c73da.es.us-central1.gcp.cloud.es.io
+    password: changeme
+    endpoint: https://not-used:9200
+    tls.insecure_skip_verify: true
 service:
   pipelines:
     logs:
       receivers: [filebeatreceiver]
       processors: []
-      exporters: [debug, elasticsearch]
+      exporters: [debug, elasticsearch]
+
+# the output parsed/used when running with -config filebeat:/path/to/file
+output:
+  elasticsearch:
+    hosts:
+      - https://localhost:9200
+    username: elastic
+    password: changeme
+    ssl.verification_mode: none