Merge branch '8.15' into mergify/bp/8.15/pr-41119

.go-version

@@ -1 +1 @@
-1.22.7
+1.22.8

.golangci.yml

@@ -124,7 +124,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -142,19 +142,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.22.7"
+    go: "1.22.8"
 
   gosec:
     excludes:

CHANGELOG.next.asciidoc

@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Removed deprecated Cylance from Beats. See <<migrate-from-deprecated-module>> for migration options. {pull}38037[38037]
 - Removed deprecated Bluecoat from Beats. See <<migrate-from-deprecated-module>> for migration options. {pull}38037[38037]
 - Introduce input/netmetrics and refactor netflow input metrics {pull}38055[38055]
+- Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099]
 
 *Heartbeat*
 
@@ -68,6 +69,19 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
+- Fix the paths in the .cmd script added to the path by the Windows MSI to point to the new C:\Program Files installation location. https://github.com/elastic/elastic-stack-installers/pull/238
+- Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561]
+- Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561]
+- Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561]
+- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982]
+- Rename the field "apache2.module.error" to "apache.module.error" in Apache error visualization. {issue}39480[39480] {pull}39481[39481]
+- Validate config of the `replace` processor {pull}40047[40047]
+- Allow port number 0 in the community ID flowhash processor {pull}40259[40259]
+- Fix handling of escaped brackets in syslog structured data. {issue}40445[40445] {pull}40446[40446]
+- Aborts all active connections for Elasticsearch output. {pull}40572[40572]
+- Closes beat Publisher on beat stop and by the Agent manager. {pull}40572[40572]
+- The journald input now restarts if there is an error/crash {issue}32782[32782] {pull}40558[40558]
+- Update Go version to 1.22.8. {pull}41139[41139]
 
 *Auditbeat*
 
@@ -97,6 +111,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Prevent GCP Pub/Sub input blockage by increasing default value of `max_outstanding_messages` {issue}35029[35029] {pull}38985[38985]
 - Updated Websocket input title to align with existing inputs {pull}39006[39006]
 - Fix replace processor handling of zero string replacement validation. {pull}40751[40751]
+- Add backup and delete for AWS S3 polling mode feature back. {pull}41071[41071]
 
 *Heartbeat*
 
@@ -109,6 +124,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Azure Monitor 429 error by causing metricbeat to retry the request again. {pull}38294[38294]
 - Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720]
 - rabbitmq/queue - Change the mapping type of `rabbitmq.queue.consumers.utilisation.pct` to `scaled_float` from `long` because the values fall within the range of `[0.0, 1.0]`. Previously, conversion to integer resulted in reporting either `0` or `1`.
+- Use namespace for GetListMetrics when exists in AWS {pull}41022[41022]
 
 *Osquerybeat*
 
@@ -137,6 +153,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Enable early event encoding in the Elasticsearch output, improving cpu and memory use {pull}38572[38572]
 - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669]
 - Update to Go 1.22.7. {pull}41018[41018]
+- Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942]
+- Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled
 
 *Auditbeat*
 
@@ -190,6 +208,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 *Heartbeat*
 
 - Added status to monitor run log report.
+- Add monitor status reporter under managed mode. {pull}41077[41077]
 
 *Metricbeat*
 

NOTICE.txt

@@ -12612,11 +12612,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.8.1
+Version: v0.9.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.8.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.9.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -13056,11 +13056,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.9.13
+Version: v0.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.9.13/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -38800,6 +38800,36 @@ Contents of probable licence file $GOMODCACHE/github.com/godror/knownpb@v0.1.0/L
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/gofrs/uuid/v5
+Version: v5.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/v5@v5.2.0/LICENSE:
+
+Copyright (C) 2013-2018 by Maxim Bublis <b@codemonkey.ru>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/golang-jwt/jwt/v4
 Version: v4.5.0

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.7
+FROM golang:1.22.8
 
 RUN \
     apt-get update \

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.22.7 as builder
+FROM golang:1.22.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.22.6 as builder
+FROM golang:1.22.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.22.7 as builder
+FROM golang:1.22.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

go.mod

@@ -2,8 +2,6 @@ module github.com/elastic/beats/v7
 
 go 1.22.0
 
-toolchain go1.22.7
-
 require (
 	cloud.google.com/go/bigquery v1.59.1
 	cloud.google.com/go/monitoring v1.18.0
@@ -200,8 +198,8 @@ require (
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
-	github.com/elastic/elastic-agent-autodiscover v0.8.1
-	github.com/elastic/elastic-agent-libs v0.9.13
+	github.com/elastic/elastic-agent-autodiscover v0.9.0
+	github.com/elastic/elastic-agent-libs v0.12.1
 	github.com/elastic/elastic-agent-system-metrics v0.10.4-0.20240826151019-9db0a02d3b85
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/elastic/go-sfdc v0.0.0-20240621062639-bcc8456508ff

go.sum

@@ -558,12 +558,12 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ebpfevents v0.6.0 h1:BrL3m7JFK7U6h2jkbk3xAWWs//IZnugCHEDds5u2v68=
 github.com/elastic/ebpfevents v0.6.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU=
-github.com/elastic/elastic-agent-autodiscover v0.8.1 h1:u6TWqh7wfevu6S4GUq4SIxYBRo4b/P5RZmx/rSvT10A=
-github.com/elastic/elastic-agent-autodiscover v0.8.1/go.mod h1:0gzGsaDCAqBfUZjuCqqWsSI60eaZ778A5tQZV72rPV0=
+github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0JojNasPgYIA47gpuuns=
+github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.13.0 h1:ENCfV5XIMmjWo9/0J7t//5N7xgm43Ktg0SyIomupRcA=
 github.com/elastic/elastic-agent-client/v7 v7.13.0/go.mod h1:h2yJHN8Q5rhfi9i6FfyPufh+StFN+UD9PYGv8blXKbE=
-github.com/elastic/elastic-agent-libs v0.9.13 h1:D1rh1s67zlkDWmixWQaNWzn+qy6DafIDPTQnLpBNBUA=
-github.com/elastic/elastic-agent-libs v0.9.13/go.mod h1:G9ljFvDE+muOOOQBf2eRituF0fE4suGkv25rfjTwY+c=
+github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0=
+github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.10.4-0.20240826151019-9db0a02d3b85 h1:d2oD8FPZ4eBiNEg8tZVLFgKtwYSA65Xq35wUianqLJU=
 github.com/elastic/elastic-agent-system-metrics v0.10.4-0.20240826151019-9db0a02d3b85/go.mod h1:3JwPa3zZJjmBYN87xwdLcFpHrUkWpR863jiYdg39sSc=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
@@ -868,6 +868,8 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM=
+github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

heartbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.7
+FROM golang:1.22.8
 
 RUN \
     apt-get update \

heartbeat/monitors/monitor.go

@@ -34,6 +34,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
 	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/management/status"
 )
 
 // ErrMonitorDisabled is returned when the monitor plugin is marked as disabled.
@@ -71,6 +72,11 @@ type Monitor struct {
 	stats plugin.RegistryRecorder
 
 	monitorStateTracker *monitorstate.Tracker
+	statusReporter      status.StatusReporter
+}
+
+func (m *Monitor) SetStatusReporter(statusReporter status.StatusReporter) {
+	m.statusReporter = statusReporter
 }
 
 // String prints a description of the monitor in a threadsafe way. It is important that this use threadsafe
@@ -175,6 +181,9 @@ func newMonitorUnsafe(
 
 		logp.L().Error(fullErr)
 		p.Jobs = []jobs.Job{func(event *beat.Event) ([]jobs.Job, error) {
+			// if statusReporter is set, as it is for running managed-mode, update the input status
+			// to failed, specifying the error
+			m.updateStatus(status.Failed, fmt.Sprintf("monitor could not be started: %s, err: %s", m.stdFields.ID, fullErr))
 			return nil, fullErr
 		}}
 
@@ -237,6 +246,7 @@ func (m *Monitor) Start() {
 
 	m.stats.StartMonitor(int64(m.endpoints))
 	m.state = MON_STARTED
+	m.updateStatus(status.Running, "")
 }
 
 // Stop stops the monitor without freeing it in global dedup
@@ -262,4 +272,11 @@ func (m *Monitor) Stop() {
 
 	m.stats.StopMonitor(int64(m.endpoints))
 	m.state = MON_STOPPED
+	m.updateStatus(status.Stopped, "")
+}
+
+func (m *Monitor) updateStatus(status status.Status, msg string) {
+	if m.statusReporter != nil {
+		m.statusReporter.UpdateStatus(status, msg)
+	}
 }

heartbeat/monitors/monitor_test.go

@@ -18,6 +18,7 @@
 package monitors
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -32,7 +33,9 @@ import (
 	"github.com/elastic/go-lookslike/testslike"
 	"github.com/elastic/go-lookslike/validator"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
+	"github.com/elastic/beats/v7/libbeat/management/status"
 )
 
 // TestMonitorBasic tests a basic config
@@ -131,3 +134,60 @@ func TestCheckInvalidConfig(t *testing.T) {
 
 	require.Error(t, checkMonitorConfig(serverMonConf, reg))
 }
+
+type MockStatusReporter struct {
+	us func(status status.Status, msg string)
+}
+
+func (sr *MockStatusReporter) UpdateStatus(status status.Status, msg string) {
+	sr.us(status, msg)
+}
+
+func TestStatusReporter(t *testing.T) {
+	confMap := map[string]interface{}{
+		"type":     "fail",
+		"urls":     []string{"http://example.net"},
+		"schedule": "@every 1ms",
+		"name":     "myName",
+		"id":       "myId",
+	}
+	cfg, err := conf.NewConfigFrom(confMap)
+	require.NoError(t, err)
+
+	reg, _, _ := mockPluginsReg()
+	pipel := &MockPipeline{}
+	monReg := monitoring.NewRegistry()
+
+	mockDegradedPluginFactory := plugin.PluginFactory{
+		Name:    "fail",
+		Aliases: []string{"failAlias"},
+		Make: func(s string, cfg *conf.C) (plugin.Plugin, error) {
+			return plugin.Plugin{}, fmt.Errorf("error plugin")
+		},
+		Stats: plugin.NewPluginCountersRecorder("fail", monReg),
+	}
+	_ = reg.Add(mockDegradedPluginFactory)
+
+	sched := scheduler.Create(1, monitoring.NewRegistry(), time.Local, nil, true)
+	defer sched.Stop()
+
+	c, err := pipel.Connect()
+	require.NoError(t, err)
+	m, err := newMonitor(cfg, reg, c, sched.Add, nil, nil)
+	require.NoError(t, err)
+
+	// Track status marked as failed during run_once execution
+	failed := false
+	m.SetStatusReporter(&MockStatusReporter{
+		us: func(s status.Status, msg string) {
+			if s == status.Failed {
+				failed = true
+			}
+		},
+	})
+	m.Start()
+
+	sched.WaitForRunOnce()
+
+	require.True(t, failed)
+}