Merge branch 'main' into feat/iron-bank-ubi-8.8

CHANGELOG.next.asciidoc

@@ -65,6 +65,7 @@ is collected by it.
 - Support build of projects outside of beats directory {pull}36126[36126]
 - Add default cgroup regex for add_process_metadata processor {pull}36484[36484] {issue}32961[32961]
 - Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471]
+- syslog processor - Fix the ability to use `when` conditions on the processor. {issue}36762[36762]
 
 
 *Auditbeat*
@@ -232,13 +233,16 @@ is collected by it.
 - Added support for new features and removed partial save mechanism in the GCS input. {issue}35847[35847] {pull}36713[36713]
 - Re-use buffers to optimise memory allocation in fingerprint mode of filestream {pull}36736[36736]
 - Allow http_endpoint input to receive PUT and PATCH requests. {pull}36734[36734]
+- Add cache processor. {pull}36786[36786]
+- Avoid unwanted publication of Azure entity records. {pull}36753[36753]
 
 *Auditbeat*
 
 *Libbeat*
 
 *Heartbeat*
 - Added status to monitor run log report.
+- Capture and log the individual connection metrics for all the lightweight monitors
 
 
 *Metricbeat*

auditbeat/auditbeat.reference.yml

@@ -1348,6 +1348,8 @@ setup.template.settings:
 
 # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for.
 # The default data stream pattern is auditbeat-%{[agent.version]}"
+# The template string `%{[agent.version]}` will resolve to the current stack version. 
+# The other possible template value is `%{[beat.name]}`.
 #setup.dsl.data_stream_pattern: "auditbeat-%{[agent.version]}"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
@@ -1362,6 +1364,7 @@ setup.template.settings:
 
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.dsl.overwrite: false
+
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

filebeat/cmd/root.go

@@ -28,6 +28,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
 
 	// Import processors.
+	_ "github.com/elastic/beats/v7/libbeat/processors/cache"
 	_ "github.com/elastic/beats/v7/libbeat/processors/timestamp"
 )
 

filebeat/filebeat.reference.yml

@@ -2444,6 +2444,8 @@ setup.template.settings:
 
 # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for.
 # The default data stream pattern is filebeat-%{[agent.version]}"
+# The template string `%{[agent.version]}` will resolve to the current stack version. 
+# The other possible template value is `%{[beat.name]}`.
 #setup.dsl.data_stream_pattern: "filebeat-%{[agent.version]}"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
@@ -2458,6 +2460,7 @@ setup.template.settings:
 
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.dsl.overwrite: false
+
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

heartbeat/heartbeat.reference.yml

@@ -1440,6 +1440,8 @@ setup.template.settings:
 
 # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for.
 # The default data stream pattern is heartbeat-%{[agent.version]}"
+# The template string `%{[agent.version]}` will resolve to the current stack version. 
+# The other possible template value is `%{[beat.name]}`.
 #setup.dsl.data_stream_pattern: "heartbeat-%{[agent.version]}"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
@@ -1454,6 +1456,7 @@ setup.template.settings:
 
 # Overwrite the lifecycle policy at startup. The default is false.
 #setup.dsl.overwrite: false
+
 # =================================== Kibana ===================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.

heartbeat/monitors/logger/logger.go

@@ -39,13 +39,16 @@ type DurationLoggable struct {
 	Mills int64 `json:"ms"`
 }
 
+type NetworkInfo map[string]interface{}
+
 type MonitorRunInfo struct {
-	MonitorID string `json:"id"`
-	Type      string `json:"type"`
-	Duration  int64  `json:"-"`
-	Steps     *int   `json:"steps,omitempty"`
-	Status    string `json:"status"`
-	Attempt   int    `json:"attempt"`
+	MonitorID   string      `json:"id"`
+	Type        string      `json:"type"`
+	Duration    int64       `json:"-"`
+	Steps       *int        `json:"steps,omitempty"`
+	Status      string      `json:"status"`
+	Attempt     int         `json:"attempt"`
+	NetworkInfo NetworkInfo `json:"network_info,omitempty"`
 }
 
 func (m *MonitorRunInfo) MarshalJSON() ([]byte, error) {
@@ -113,12 +116,14 @@ func extractRunInfo(event *beat.Event) (*MonitorRunInfo, error) {
 		return nil, fmt.Errorf("logErrors: %+v", errors)
 	}
 
+	networkInfo := extractNetworkInfo(event, monType.(string))
 	monitor := MonitorRunInfo{
-		MonitorID: monitorID.(string),
-		Type:      monType.(string),
-		Duration:  durationUs.(int64),
-		Status:    status.(string),
-		Attempt:   attempt,
+		MonitorID:   monitorID.(string),
+		Type:        monType.(string),
+		Duration:    durationUs.(int64),
+		Status:      status.(string),
+		Attempt:     attempt,
+		NetworkInfo: networkInfo,
 	}
 
 	sc, _ := event.Meta.GetValue(META_STEP_COUNT)
@@ -130,6 +135,29 @@ func extractRunInfo(event *beat.Event) (*MonitorRunInfo, error) {
 	return &monitor, nil
 }
 
+func extractNetworkInfo(event *beat.Event, monitorType string) NetworkInfo {
+	// Only relevant for lightweight monitors
+	if monitorType == "browser" {
+		return nil
+	}
+
+	fields := []string{
+		"resolve.ip", "resolve.rtt.us", "tls.rtt.handshake.us", "icmp.rtt.us",
+		"tcp.rtt.connect.us", "tcp.rtt.validate.us", "http.rtt.content.us", "http.rtt.validate.us",
+		"http.rtt.validate_body.us", "http.rtt.write_request.us", "http.rtt.response_header.us",
+		"http.rtt.total.us", "socks5.rtt.connect.us",
+	}
+	networkInfo := make(NetworkInfo)
+	for _, field := range fields {
+		value, err := event.GetValue(field)
+		if err == nil && value != nil {
+			networkInfo[field] = value
+		}
+	}
+
+	return networkInfo
+}
+
 func LogRun(event *beat.Event) {
 	monitor, err := extractRunInfo(event)
 	if err != nil {

heartbeat/monitors/logger/logger_test.go

@@ -34,43 +34,111 @@ import (
 	"github.com/elastic/elastic-agent-libs/mapstr"
 )
 
-func TestLogRun(t *testing.T) {
-	core, observed := observer.New(zapcore.InfoLevel)
-	SetLogger(logp.NewLogger("t", zap.WrapCore(func(in zapcore.Core) zapcore.Core {
-		return zapcore.NewTee(in, core)
-	})))
-
-	durationUs := int64(5000 * time.Microsecond)
-	steps := 1337
-
-	fields := mapstr.M{
-		"monitor.id":          "b0",
-		"monitor.duration.us": durationUs,
-		"monitor.type":        "browser",
-		"monitor.status":      "down",
-		"summary":             jobsummary.NewJobSummary(1, 1, "abc"),
+func generateFakeNetworkInfo() NetworkInfo {
+	networkInfo := NetworkInfo{
+		// All network info available in HB documentation
+		"resolve.ip":                  "192.168.1.254",
+		"resolve.rtt.us":              123,
+		"tls.rtt.handshake.us":        456,
+		"icmp.rtt.us":                 111,
+		"tcp.rtt.connect.us":          789,
+		"tcp.rtt.validate.us":         1234,
+		"http.rtt.content.us":         4567,
+		"http.rtt.validate.us":        7890,
+		"http.rtt.validate_body.us":   12345,
+		"http.rtt.write_request.us":   45678,
+		"http.rtt.response_header.us": 78901,
+		"http.rtt.total.us":           123456,
+		"socks5.rtt.connect.us":       789012,
 	}
 
-	event := beat.Event{Fields: fields}
-	eventext.SetMeta(&event, META_STEP_COUNT, steps)
+	return networkInfo
+}
 
-	LogRun(&event)
+func TestLogRun(t *testing.T) {
+	t.Run("should log the monitor completion", func(t *testing.T) {
+		core, observed := observer.New(zapcore.InfoLevel)
+		SetLogger(logp.NewLogger("t", zap.WrapCore(func(in zapcore.Core) zapcore.Core {
+			return zapcore.NewTee(in, core)
+		})))
 
-	observedEntries := observed.All()
-	require.Len(t, observedEntries, 1)
-	assert.Equal(t, "Monitor finished", observedEntries[0].Message)
+		durationUs := int64(5000 * time.Microsecond)
+		steps := 1337
+		fields := mapstr.M{
+			"monitor.id":          "b0",
+			"monitor.duration.us": durationUs,
+			"monitor.type":        "browser",
+			"monitor.status":      "down",
+			"summary":             jobsummary.NewJobSummary(1, 1, "abc"),
+		}
 
-	expectedMonitor := MonitorRunInfo{
-		MonitorID: "b0",
-		Type:      "browser",
-		Duration:  durationUs,
-		Status:    "down",
-		Steps:     &steps,
-		Attempt:   1,
-	}
+		event := beat.Event{Fields: fields}
+		eventext.SetMeta(&event, META_STEP_COUNT, steps)
+
+		LogRun(&event)
+
+		observedEntries := observed.All()
+		require.Len(t, observedEntries, 1)
+		assert.Equal(t, "Monitor finished", observedEntries[0].Message)
+
+		expectedMonitor := MonitorRunInfo{
+			MonitorID: "b0",
+			Type:      "browser",
+			Duration:  durationUs,
+			Status:    "down",
+			Steps:     &steps,
+			Attempt:   1,
+		}
+
+		assert.ElementsMatch(t, []zap.Field{
+			logp.Any("event", map[string]string{"action": ActionMonitorRun}),
+			logp.Any("monitor", &expectedMonitor),
+		}, observedEntries[0].Context)
+	})
+
+	t.Run("should log network information if available", func(t *testing.T) {
+		core, observed := observer.New(zapcore.InfoLevel)
+		SetLogger(logp.NewLogger("t", zap.WrapCore(func(in zapcore.Core) zapcore.Core {
+			return zapcore.NewTee(in, core)
+		})))
+
+		durationUs := int64(5000 * time.Microsecond)
+		steps := 1337
+		fields := mapstr.M{
+			"monitor.id":          "b0",
+			"monitor.duration.us": durationUs,
+			"monitor.type":        "http",
+			"monitor.status":      "down",
+			"summary":             jobsummary.NewJobSummary(1, 1, "abc"),
+		}
+		networkInfo := generateFakeNetworkInfo()
+		// Add network info to the event
+		for key, value := range networkInfo {
+			fields[key] = value
+		}
+
+		event := beat.Event{Fields: fields}
+		eventext.SetMeta(&event, META_STEP_COUNT, steps)
+
+		LogRun(&event)
+
+		observedEntries := observed.All()
+		require.Len(t, observedEntries, 1)
+		assert.Equal(t, "Monitor finished", observedEntries[0].Message)
+
+		expectedMonitor := MonitorRunInfo{
+			MonitorID:   "b0",
+			Type:        "http",
+			Duration:    durationUs,
+			Status:      "down",
+			Steps:       &steps,
+			Attempt:     1,
+			NetworkInfo: networkInfo,
+		}
 
-	assert.ElementsMatch(t, []zap.Field{
-		logp.Any("event", map[string]string{"action": ActionMonitorRun}),
-		logp.Any("monitor", &expectedMonitor),
-	}, observedEntries[0].Context)
+		assert.ElementsMatch(t, []zap.Field{
+			logp.Any("event", map[string]string{"action": ActionMonitorRun}),
+			logp.Any("monitor", &expectedMonitor),
+		}, observedEntries[0].Context)
+	})
 }

heartbeat/monitors/wrappers/wrappers_test.go

@@ -140,11 +140,12 @@ func TestSimpleJob(t *testing.T) {
 			require.NoError(t, err)
 
 			expectedMonitor := logger.MonitorRunInfo{
-				MonitorID: testMonFields.ID,
-				Type:      testMonFields.Type,
-				Duration:  durationUs.(int64),
-				Status:    "up",
-				Attempt:   1,
+				MonitorID:   testMonFields.ID,
+				Type:        testMonFields.Type,
+				Duration:    durationUs.(int64),
+				Status:      "up",
+				Attempt:     1,
+				NetworkInfo: logger.NetworkInfo{},
 			}
 			require.ElementsMatch(t, []zap.Field{
 				logp.Any("event", map[string]string{"action": logger.ActionMonitorRun}),

libbeat/_meta/config/setup.dsl.reference.yml.tmpl

@@ -8,6 +8,8 @@
 
 # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for.
 # The default data stream pattern is {{.BeatName}}-%{[agent.version]}"
+# The template string `%{[agent.version]}` will resolve to the current stack version. 
+# The other possible template value is `%{[beat.name]}`.
 #setup.dsl.data_stream_pattern: "{{.BeatName}}-%{[agent.version]}"
 
 # The path to a JSON file that contains a lifecycle policy configuration. Used
@@ -21,4 +23,4 @@
 #setup.dsl.check_exists: true
 
 # Overwrite the lifecycle policy at startup. The default is false.
-#setup.dsl.overwrite: false
+#setup.dsl.overwrite: false

libbeat/processors/syslog/syslog.go

@@ -89,6 +89,7 @@ func init() {
 				"ignore_missing",
 				"ignore_failure",
 				"tag",
+				"when",
 			),
 		),
 	)

metricbeat/docker-compose.yml

@@ -17,11 +17,11 @@ services:
 
   # Used by base tests
   elasticsearch:
-    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.9.1}-1
+    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.10.2}-1
     build:
       context: ./module/elasticsearch/_meta
       args:
-        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.9.1}
+        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.10.2}
     environment:
       - "ES_JAVA_OPTS=-Xms256m -Xmx256m"
       - "transport.host=127.0.0.1"
@@ -38,11 +38,11 @@ services:
 
   # Used by base tests
   kibana:
-    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.9.1}-1
+    image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.10.2}-1
     build:
       context: ./module/kibana/_meta
       args:
-        KIBANA_VERSION: ${KIBANA_VERSION:-8.9.1}
+        KIBANA_VERSION: ${KIBANA_VERSION:-8.10.2}
     healthcheck:
       test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"]
       retries: 600
@@ -53,11 +53,11 @@ services:
 
   # Used by base tests
   metricbeat:
-    image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.9.1}-1
+    image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.10.2}-1
     build:
       context: ./module/beat/_meta
       args:
-        BEAT_VERSION: ${BEAT_VERSION:-8.9.1}
+        BEAT_VERSION: ${BEAT_VERSION:-8.10.2}
     command: '-e'
     ports:
       - 5066:5066