Merge branch 'main' into modern-session-view

.github/CODEOWNERS

@@ -123,6 +123,7 @@ CHANGELOG*
 /x-pack/filebeat/input/httpjson/ @elastic/security-service-integrations
 /x-pack/filebeat/input/internal/httplog @elastic/security-service-integrations
 /x-pack/filebeat/input/internal/httpmon @elastic/security-service-integrations
+/x-pack/filebeat/input/internal/private @elastic/security-service-integrations
 /x-pack/filebeat/input/lumberjack/ @elastic/security-service-integrations
 /x-pack/filebeat/input/netflow/ @elastic/sec-deployment-and-devices
 /x-pack/filebeat/input/o365audit/ @elastic/security-service-integrations

CHANGELOG-developer.next.asciidoc

@@ -207,6 +207,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Added filebeat debug histograms for s3 object size and events per processed s3 object. {pull}40775[40775]
 - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
 - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
+- Add field redaction package. {pull}40997[40997]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Added `container.image.name` to `journald` Filebeat input's Docker-specific translated fields. {pull}40450[40450]
 - Change log.file.path field in awscloudwatch input to nested object. {pull}41099[41099]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
+- System module events now contain `input.type: systemlogs` instead of `input.type: log` when harvesting log files. {pull}41061[41061]
 
 
 *Heartbeat*
@@ -236,7 +237,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions.
 - Update to Go 1.22.7. {pull}41018[41018]
 - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942]
-
+- Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled
 
 *Auditbeat*
 
@@ -324,6 +325,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improved GCS input documentation. {pull}41143[41143]
 - Add CSV decoding capacity to azureblobstorage input {pull}40978[40978]
 - Add CSV decoding capacity to gcs input {pull}40979[40979]
+- Jounrald input now supports filtering by facilities {pull}41061[41061]
+- System module now supports reading from jounrald. {pull}41061[41061]
 
 *Auditbeat*
 

NOTICE.txt

@@ -12575,11 +12575,11 @@ various licenses:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.8.2
+Version: v0.9.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.8.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.9.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -205,7 +205,7 @@ RUN cd /usr/share/heartbeat/.node \
     && curl ${NODE_DOWNLOAD_URL} | tar -xJ --strip 1 -C node \
     && chmod ug+rwX -R $NODE_PATH
 
-# Install synthetics as a regular user, installing npm deps as root odesn't work
+# Install synthetics as a regular user, installing npm deps as root doesn't work
 RUN chown -R {{ .user }} $NODE_PATH
 USER {{ .user }}
 # If this fails dump the NPM logs
@@ -227,7 +227,11 @@ done; \
 (exit $exit_code)
 {{- end }}
 
-USER {{ .user }}
+{{- if eq .user "root" }}
+USER 0
+{{- else }}
+USER 1000
+{{- end }}
 
 {{- range $i, $port := .ExposePorts }}
 EXPOSE {{ $port }}

filebeat/docs/include/use-journald.asciidoc

@@ -0,0 +1,12 @@
+*`var.use_journald`*::
+
+A boolean that when set to `true` will read logs from Journald. When
+Journald is used all events contain the tag `journald`
+
+*`var.use_files`*::
+
+A boolean that when set to `true` will read logs from the log files
+defined by `vars.paths`.
+
+If neither `var.use_journald` nor `var.use_files` are set (or both are
+`false`) {beatname_uc} will auto-detect the source for the logs.

filebeat/docs/include/var-paths.asciidoc

@@ -6,4 +6,4 @@ are also supported here. For example, you can use wildcards to fetch all files
 from a predefined level of subdirectories: `/path/to/log/*/*.log`. This
 fetches all `.log` files from the subfolders of `/path/to/log`. It does not
 fetch log files from the `/path/to/log` folder itself. If this setting is left
-empty, {beatname_uc} will choose log paths based on your operating system.
+empty, {beatname_uc} will choose log paths based on your operating system.

filebeat/docs/inputs/input-journald.asciidoc

@@ -169,6 +169,13 @@ Valid transports:
 * stdout: messages from a service's standard output or error output
 * kernel: messages from the kernel
 
+[float]
+[id="{beatname_lc}-input-{type}-facilities"]
+==== `facilities`
+
+Filter entries by facilities, facilities must be specified using their
+numeric code.
+
 [float]
 [id="{beatname_lc}-input-{type}-include-matches"]
 ==== `include_matches`

filebeat/docs/modules/system.asciidoc

@@ -23,7 +23,7 @@ include::../include/gs-link.asciidoc[]
 === Compatibility
 
 This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
-macOS Sierra.
+macOS Sierra. For Debian 12 Journald is used to read the system logs.
 
 This module is not available for Windows.
 
@@ -65,11 +65,15 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/use-journald.asciidoc[]
+
 [float]
 ==== `auth` fileset settings
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/use-journald.asciidoc[]
+
 *`var.tags`*::
 
 A list of tags to include in events. Including `forwarded` indicates that the

filebeat/filebeat.reference.yml

@@ -21,7 +21,18 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Input configuration (advanced). Any input configuration option
+    # Force using journald to collect system logs
+    #var.use_journald: true|false
+
+    # Force using log files to collect system logs
+    #var.use_files: true|false
+
+    # If use_journald and use_files are false, then
+    # Filebeat will autodetect whether use to journald
+    # to collect system logs.
+
+    # Input configuration (advanced).
+    # Any input configuration option
     # can be added under this section.
     #input:
 
@@ -33,6 +44,23 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Force using journald to collect system logs
+    #var.use_journald: true|false
+
+    # Force using log files to collect system logs
+    #var.use_files: true|false
+
+    # If use_journald and use_files are false, then
+    # Filebeat will autodetect whether use to journald
+    # to collect system logs.
+
+    # A list of tags to include in events. Including 'forwarded'
+    # indicates that the events did not originate on this host and
+    # causes host.name to not be added to events. Include
+    # 'preserve_orginal_event' causes the pipeline to retain the raw log
+    # in event.original. Defaults to [].
+    #var.tags: []
+
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:

filebeat/fileset/fileset.go

@@ -24,7 +24,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -143,11 +142,11 @@ type ProcessorRequirement struct {
 func (fs *Fileset) readManifest() (*manifest, error) {
 	cfg, err := common.LoadFile(filepath.Join(fs.modulePath, fs.name, "manifest.yml"))
 	if err != nil {
-		return nil, fmt.Errorf("Error reading manifest file: %v", err)
+		return nil, fmt.Errorf("Error reading manifest file: %w", err)
 	}
 	manifest, err := newManifest(cfg)
 	if err != nil {
-		return nil, fmt.Errorf("Error unpacking manifest: %v", err)
+		return nil, fmt.Errorf("Error unpacking manifest: %w", err)
 	}
 	return manifest, nil
 }
@@ -183,7 +182,7 @@ func (fs *Fileset) evaluateVars(info beat.Info) (map[string]interface{}, error)
 
 		vars[name], err = resolveVariable(vars, value)
 		if err != nil {
-			return nil, fmt.Errorf("Error resolving variables on %s: %v", name, err)
+			return nil, fmt.Errorf("Error resolving variables on %s: %w", name, err)
 		}
 	}
 
@@ -246,7 +245,7 @@ func resolveVariable(vars map[string]interface{}, value interface{}) (interface{
 			if ok {
 				transf, err := ApplyTemplate(vars, s, false)
 				if err != nil {
-					return nil, fmt.Errorf("array: %v", err)
+					return nil, fmt.Errorf("array: %w", err)
 				}
 				transformed = append(transformed, transf)
 			} else {
@@ -322,33 +321,35 @@ func getTemplateFunctions(vars map[string]interface{}) (template.FuncMap, error)
 // getBuiltinVars computes the supported built in variables and groups them
 // in a dictionary
 func (fs *Fileset) getBuiltinVars(info beat.Info) (map[string]interface{}, error) {
-	host, err := os.Hostname()
-	if err != nil || len(host) == 0 {
+	osHost, err := os.Hostname()
+	if err != nil || len(osHost) == 0 {
 		return nil, fmt.Errorf("Error getting the hostname: %w", err)
 	}
-	split := strings.SplitN(host, ".", 2)
+	split := strings.SplitN(osHost, ".", 2)
 	hostname := split[0]
 	domain := ""
 	if len(split) > 1 {
 		domain = split[1]
 	}
 
-	return map[string]interface{}{
+	vars := map[string]interface{}{
 		"prefix":      info.IndexPrefix,
 		"hostname":    hostname,
 		"domain":      domain,
 		"module":      fs.mname,
 		"fileset":     fs.name,
 		"beatVersion": info.Version,
-	}, nil
+	}
+
+	return vars, nil
 }
 
 func (fs *Fileset) getInputConfig() (*conf.C, error) {
 	path, err := ApplyTemplate(fs.vars, fs.manifest.Input, false)
 	if err != nil {
 		return nil, fmt.Errorf("Error expanding vars on the input path: %w", err)
 	}
-	contents, err := ioutil.ReadFile(filepath.Join(fs.modulePath, fs.name, path))
+	contents, err := os.ReadFile(filepath.Join(fs.modulePath, fs.name, path))
 	if err != nil {
 		return nil, fmt.Errorf("Error reading input file %s: %w", path, err)
 	}
@@ -434,7 +435,7 @@ func (fs *Fileset) GetPipelines(esVersion version.V) (pipelines []pipeline, err
 			return nil, fmt.Errorf("Error expanding vars on the ingest pipeline path: %w", err)
 		}
 
-		strContents, err := ioutil.ReadFile(filepath.Join(fs.modulePath, fs.name, path))
+		strContents, err := os.ReadFile(filepath.Join(fs.modulePath, fs.name, path))
 		if err != nil {
 			return nil, fmt.Errorf("Error reading pipeline file %s: %w", path, err)
 		}
@@ -458,7 +459,11 @@ func (fs *Fileset) GetPipelines(esVersion version.V) (pipelines []pipeline, err
 			if err != nil {
 				return nil, fmt.Errorf("Failed to sanitize the YAML pipeline file: %s: %w", path, err)
 			}
-			content = newContent.(map[string]interface{})
+			var ok bool
+			content, ok = newContent.(map[string]interface{})
+			if !ok {
+				return nil, errors.New("cannot convert newContent to map[string]interface{}")
+			}
 		default:
 			return nil, fmt.Errorf("Unsupported extension '%s' for pipeline file: %s", extension, path)
 		}

filebeat/input/default-inputs/inputs_linux.go

@@ -19,6 +19,7 @@ package inputs
 
 import (
 	"github.com/elastic/beats/v7/filebeat/input/journald"
+	"github.com/elastic/beats/v7/filebeat/input/systemlogs"
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
 	cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor"
 	"github.com/elastic/beats/v7/libbeat/beat"
@@ -37,6 +38,7 @@ func osInputs(info beat.Info, log *logp.Logger, components osComponents) []v2.Pl
 	zeroPlugin := v2.Plugin{}
 	if journald := journald.Plugin(log, components); journald != zeroPlugin {
 		plugins = append(plugins, journald)
+		plugins = append(plugins, systemlogs.PluginV2(log, components))
 	}
 
 	return plugins

filebeat/input/filestream/fswatch_test.go

@@ -36,6 +36,7 @@ import (
 )
 
 func TestFileWatcher(t *testing.T) {
+	t.Skip("Flaky test: https://github.com/elastic/beats/issues/41209")
 	dir := t.TempDir()
 	paths := []string{filepath.Join(dir, "*.log")}
 	cfgStr := `

filebeat/input/filestream/internal/task/group_test.go

@@ -67,6 +67,7 @@ func TestNewGroup(t *testing.T) {
 }
 
 func TestGroup_Go(t *testing.T) {
+	t.Skip("Flaky tests: https://github.com/elastic/beats/issues/41218")
 	t.Run("don't run more than limit goroutines", func(t *testing.T) {
 		done := make(chan struct{})
 		defer close(done)

filebeat/input/journald/README.md

@@ -0,0 +1,57 @@
+# Journald input
+
+The Journald input reads journal entries by calling `journalctl`.
+
+## Adding entries to the journal
+The easiest way to add entries to the journal is to use `systemd-cat`:
+```
+root@vagrant-debian-12:~/filebeat# echo "Hello Journal!" | systemd-cat
+root@vagrant-debian-12:~/filebeat# journalctl -n 1
+Oct 02 04:17:01 vagrant-debian-12 CRON[1912]: pam_unix(cron:session): session closed for user root
+```
+
+The syslog identifier can be specified with the `-t` parameter:
+```
+root@vagrant-debian-12:~/filebeat# echo "Hello Journal!" | systemd-cat -t my-test
+root@vagrant-debian-12:~/filebeat# journalctl -n 1
+Oct 02 04:17:50 vagrant-debian-12 my-test[1924]: Hello Journal!
+```
+
+## Crafting a journal file
+The easiest way to craft a journal file with the entries you need is
+to use
+[`systemd-journald-remote`](https://www.freedesktop.org/software/systemd/man/latest/systemd-journal-remote.service.html).
+First we need to export some entries to a file:
+```
+root@vagrant-debian-12:~/filebeat# journalctl -g "Hello" -o export >export
+```
+One good thing of the `-o export` is that you can just concatenate the
+output of any number of runs and the result will be a valid file.
+
+Then you can use `systemd-journald-remote` to generate the journal
+file:
+```
+root@vagrant-debian-12:~/filebeat# /usr/lib/systemd/systemd-journal-remote -o example.journal export
+Finishing after writing 2 entries
+``
+Or you can run as a one liner:
+```
+root@vagrant-debian-12:~/filebeat# journalctl -g "Hello" -o export | /usr/lib/systemd/systemd-journal-remote -o example.journal -
+```
+
+Then you can read the newly created file:
+```
+root@vagrant-debian-12:~/filebeat# journalctl --file ./example.journal
+Oct 02 04:16:54 vagrant-debian-12 unknown[1908]: Hello Journal!
+Oct 02 04:17:50 vagrant-debian-12 my-test[1924]: Hello Journal!
+root@vagrant-debian-12:~/filebeat# 
+```
+
+Bear in mind that `systemd-journal-remote` will **append** to the
+output file.
+
+## References
+- https://systemd.io/JOURNAL_NATIVE_PROTOCOL/
+- https://www.freedesktop.org/software/systemd/man/latest/journalctl.html
+- https://www.freedesktop.org/software/systemd/man/latest/systemd-cat.html
+- https://www.freedesktop.org/software/systemd/man/latest/systemd-journal-remote.service.html

filebeat/input/journald/config.go

@@ -63,6 +63,9 @@ type config struct {
 	// SaveRemoteHostname defines if the original source of the entry needs to be saved.
 	SaveRemoteHostname bool `config:"save_remote_hostname"`
 
+	// Facility is a list of facilities to filter journal messages
+	Facilities []int `config:"facilities"`
+
 	// Parsers configuration
 	Parsers parser.Config `config:",inline"`
 }