Merge branch 'main' into bugfix/libbeat/aws-assume-role-session-cache

.buildkite/pull-requests.json

@@ -127,6 +127,22 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "xpack-elastic-agent",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/test elastic-agent$",
+            "always_trigger_comment_regex": "^/test elastic-agent$",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": ["^xpack/elastic-agent/README.md", "^xpack/elastic-agent/docs/.*", "^xpack/elastic-agent/devtools/.*" ],
+            "always_require_ci_on_changed": ["^xpack/elastic-agent/.*", ".buildkite/xpack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
         }
     ]
 }

.buildkite/xpack/elastic-agent/pipeline.xpack.elastic-agent.yml

@@ -0,0 +1,6 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+# This pipeline is only for 7.17 branch. See catalog-info.yml
+steps:
+  - label: "Example test"
+    command: echo "Hello!"

.github/workflows/opentelemetry.yml

@@ -1,46 +1,16 @@
+---
+# Look up results at https://ela.st/oblt-ci-cd-stats.
+# There will be one service per GitHub repository, including the org name, and one Transaction per Workflow.
 name: OpenTelemetry Export Trace
 
 on:
   workflow_run:
-    workflows:
-      - bump-elastic-stack-snapshot
-      - bump-golang
-      - check-auditbeat
-      - check-default
-      - check-dev-tools
-      - check-docs
-      - check-filebeat
-      - check-heartbeat
-      - check-libbeat
-      - check-metricbeat
-      - check-packetbeat
-      - check-winlogbeat
-      - check-x-pack-auditbeat
-      - check-x-pack-dockerlogbeat
-      - check-x-pack-filebeat
-      - check-x-pack-functionbeat
-      - check-x-pack-heartbeat
-      - check-x-pack-libbeat
-      - check-x-pack-metricbeat
-      - check-x-pack-osquerybeat
-      - check-x-pack-packetbeat
-      - check-x-pack-winlogbeat
-      - golangci-lint
-      - notify-stalled-snapshots
-      - auditbeat
-      - filebeat
-      - heartbeat
-      - metricbeat
-      - packetbeat
-      - x-pack-auditbeat
-      - x-pack-filebeat
-      - x-pack-functionbeat
-      - x-pack-heartbeat
-      - x-pack-metricbeat
-      - x-pack-osquerybeat
-      - x-pack-packetbeat
+    workflows: [ "*" ]
     types: [completed]
 
+permissions:
+  contents: read
+
 jobs:
   otel-export-trace:
     runs-on: ubuntu-latest

CHANGELOG.asciidoc

@@ -15,9 +15,9 @@ Performance regression in AWS S3 inputs using SQS notification.
 
 In 8.12 the default memory queue flush interval was raised from 1 second to 10 seconds. In many configurations this improves performance because it allows the output to batch more events per round trip, which improves efficiency. However, the SQS input has an extra bottleneck that interacts badly with the new value. For more details see {issue}37754[37754].
 
-If you are using the Elasticsearch output, and your output configuration uses a performance preset, switch it to `preset: latency`. If you use no preset or use `preset: custom`, then set `queue.mem.flush.timeout: 1` in your queue or output configuration.
+If you are using the Elasticsearch output, and your output configuration uses a performance preset, switch it to `preset: latency`. If you use no preset or use `preset: custom`, then set `queue.mem.flush.timeout: 1s` in your queue or output configuration.
 
-If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1` in your queue or output configuration.
+If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1s` in your queue or output configuration.
 
 ==== Breaking changes
 

CHANGELOG.next.asciidoc

@@ -206,6 +206,8 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 *Packetbeat*
 
 - Bump Windows Npcap version to v1.79. {pull}37733[37733]
+- Add metrics for TCP flags. {issue}36992[36992] {pull}36975[36975]
+- Add support for pipeline loading. {pull}37291[37291]
 
 *Packetbeat*
 

catalog-info.yaml

@@ -377,3 +377,45 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-beats-xpack-elastic-agent
+  description: "Beats xpack elastic agent"
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/beats-xpack-elastic-agent
+
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-xpack-elastic-agent
+      description: "Beats xpack elastic agent pipeline"
+    spec:
+      branch_configuration: "7.17"
+      pipeline_file: ".buildkite/xpack/elastic-agent/pipeline.xpack.elastic-agent.yml"
+      provider_settings:
+        build_pull_request_forks: false
+        build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
+        build_tags: true
+        filter_enabled: true
+        filter_condition: >-
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+      repository: elastic/beats
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*"
+      skip_intermediate_builds: true
+      skip_intermediate_builds_branch_filter: "!main !7.17 !8.*"      
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

packetbeat/_meta/config/beat.reference.yml.tmpl

@@ -78,6 +78,11 @@ packetbeat.interfaces.internal_networks:
 # can stay enabled even after beat is shut down.
 #packetbeat.interfaces.auto_promisc_mode: true
 
+# By default Ingest pipelines are not updated if a pipeline with the same ID
+# already exists. If this option is enabled Packetbeat overwrites pipelines
+# every time a new Elasticsearch connection is established.
+#packetbeat.overwrite_pipelines: false
+
 {{- template "windows_npcap.yml.tmpl" .}}
 
 {{header "Flows"}}

packetbeat/beater/packetbeat.go

@@ -25,13 +25,16 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common/reload"
+	"github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
 	"github.com/elastic/beats/v7/libbeat/management"
 	"github.com/elastic/beats/v7/libbeat/monitoring/inputmon"
+	"github.com/elastic/beats/v7/libbeat/outputs/elasticsearch"
 	conf "github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/service"
 
 	"github.com/elastic/beats/v7/packetbeat/config"
+	"github.com/elastic/beats/v7/packetbeat/module"
 	"github.com/elastic/beats/v7/packetbeat/protos"
 
 	// Add packetbeat default processors
@@ -80,10 +83,11 @@ func initialConfig() config.Config {
 
 // Beater object. Contains all objects needed to run the beat
 type packetbeat struct {
-	config   *conf.C
-	factory  *processorFactory
-	done     chan struct{}
-	stopOnce sync.Once
+	config             *conf.C
+	factory            *processorFactory
+	overwritePipelines bool
+	done               chan struct{}
+	stopOnce           sync.Once
 }
 
 // New returns a new Packetbeat beat.Beater.
@@ -98,15 +102,35 @@ func New(b *beat.Beat, rawConfig *conf.C) (beat.Beater, error) {
 		return nil, err
 	}
 
+	var overwritePipelines bool
+	if !b.Manager.Enabled() {
+		// Pipeline overwrite is only enabled on standalone packetbeat
+		// since pipelines are managed by fleet otherwise.
+		config, err := configurator(rawConfig)
+		if err != nil {
+			return nil, err
+		}
+		overwritePipelines = config.OverwritePipelines
+		b.OverwritePipelinesCallback = func(esConfig *conf.C) error {
+			esClient, err := eslegclient.NewConnectedClient(esConfig, "Packetbeat")
+			if err != nil {
+				return err
+			}
+			_, err = module.UploadPipelines(b.Info, esClient, overwritePipelines)
+			return err
+		}
+	}
+
 	return &packetbeat{
-		config:  rawConfig,
-		factory: factory,
-		done:    make(chan struct{}),
+		config:             rawConfig,
+		factory:            factory,
+		overwritePipelines: overwritePipelines,
+		done:               make(chan struct{}),
 	}, nil
 }
 
 // Run starts the packetbeat network capture, decoding and event publication, sending
-// events to b.Publisher. If b is mananaged, packetbeat is registered with the
+// events to b.Publisher. If b is managed, packetbeat is registered with the
 // reload.Registry and handled by fleet. Otherwise it is run until cancelled or a
 // fatal error.
 func (pb *packetbeat) Run(b *beat.Beat) error {
@@ -138,11 +162,28 @@ func (pb *packetbeat) Run(b *beat.Beat) error {
 	}
 
 	if !b.Manager.Enabled() {
+		if b.Config.Output.Name() == "elasticsearch" {
+			_, err := elasticsearch.RegisterConnectCallback(func(esClient *eslegclient.Connection) error {
+				_, err := module.UploadPipelines(b.Info, esClient, pb.overwritePipelines)
+				return err
+			})
+			if err != nil {
+				return err
+			}
+		} else {
+			logp.L().Warn(pipelinesWarning)
+		}
+
 		return pb.runStatic(b, pb.factory)
 	}
 	return pb.runManaged(b, pb.factory)
 }
 
+const pipelinesWarning = "Packetbeat is unable to load the ingest pipelines for the configured" +
+	" modules because the Elasticsearch output is not configured/enabled. If you have" +
+	" already loaded the ingest pipelines or are using Logstash pipelines, you" +
+	" can ignore this warning."
+
 // runStatic constructs a packetbeat runner and starts it, returning on cancellation
 // or the first fatal error.
 func (pb *packetbeat) runStatic(b *beat.Beat, factory *processorFactory) error {

packetbeat/config/config.go

@@ -33,14 +33,15 @@ import (
 var errFanoutGroupAFPacketOnly = errors.New("fanout_group is only valid with af_packet type")
 
 type Config struct {
-	Interface       *InterfaceConfig   `config:"interfaces"`
-	Interfaces      []InterfaceConfig  `config:"interfaces"`
-	Flows           *Flows             `config:"flows"`
-	Protocols       map[string]*conf.C `config:"protocols"`
-	ProtocolsList   []*conf.C          `config:"protocols"`
-	Procs           procs.ProcsConfig  `config:"procs"`
-	IgnoreOutgoing  bool               `config:"ignore_outgoing"`
-	ShutdownTimeout time.Duration      `config:"shutdown_timeout"`
+	Interface          *InterfaceConfig   `config:"interfaces"`
+	Interfaces         []InterfaceConfig  `config:"interfaces"`
+	Flows              *Flows             `config:"flows"`
+	Protocols          map[string]*conf.C `config:"protocols"`
+	ProtocolsList      []*conf.C          `config:"protocols"`
+	Procs              procs.ProcsConfig  `config:"procs"`
+	IgnoreOutgoing     bool               `config:"ignore_outgoing"`
+	ShutdownTimeout    time.Duration      `config:"shutdown_timeout"`
+	OverwritePipelines bool               `config:"overwrite_pipelines"` // Only used by standalone Packetbeat.
 }
 
 // FromStatic initializes a configuration given a config.C

packetbeat/magefile.go

@@ -29,19 +29,20 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 	packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage"
 
-	// mage:import
+	//mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
-	// mage:import
+	//mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests"
-	// mage:import
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/test"
 )
 
 func init() {
 	common.RegisterCheckDeps(Update)
 	unittest.RegisterPythonTestDeps(packetbeat.FieldsYML, Dashboards)
+	packetbeat.SelectLogic = devtools.OSSProject
 
 	devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch."
 }