From 8a707127fb2b7646ab72ef088d60a6938775fbd6 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 13 Oct 2023 01:58:11 -0400 Subject: [PATCH 01/50] chore: Update snapshot.yml (#36834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index da9071fa620c..0bf8c5252e07 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-3a221f77-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-1a1295ff-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-3a221f77-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-1a1295ff-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-3a221f77-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-1a1295ff-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From df4d550512a5fbe71e685c5b4baf7e25b0457702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:02:37 +0200 Subject: [PATCH 02/50] [Heartbeat] Retry on no previous state (#36842) * Add empty state to retry cond * Remove unwanted test * Add changelog * address review --------- Co-authored-by: vigneshshanmugam --- CHANGELOG.next.asciidoc | 1 + .../wrappers/summarizer/plugstatestat.go | 6 +- .../wrappers/summarizer/summarizer_test.go | 10 +- heartbeat/monitors/wrappers/wrappers_test.go | 128 ------------------ 4 files changed, 9 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8ba973c8c63c..c4ec867eaf39 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -114,6 +114,7 @@ is collected by it. *Heartbeat* - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] +- Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842] *Metricbeat* diff --git a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go index 9567a666ad6d..4acfee4dc361 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go +++ b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go @@ -150,12 +150,12 @@ func (ssp *commonSSP) BeforeSummary(event *beat.Event) BeforeSummaryActions { lastStatus := ssp.stateTracker.GetCurrentStatus(ssp.sf) curCheckDown := ssp.js.Status == monitorstate.StatusDown - lastStateUp := ssp.stateTracker.GetCurrentStatus(ssp.sf) == monitorstate.StatusUp + lastStateUpOrEmpty := lastStatus == monitorstate.StatusUp || lastStatus == monitorstate.StatusEmpty hasAttemptsRemaining := ssp.js.Attempt < ssp.js.MaxAttempts // retry if... retry := curCheckDown && // the current check is down - lastStateUp && // we were previously up, if we were previously down we just check once + lastStateUpOrEmpty && // we were previously up or had no previous state, if we were previously down we just check once hasAttemptsRemaining // and we are configured to actually make multiple attempts // if we aren't retrying this is the final attempt ssp.js.FinalAttempt = !retry @@ -174,7 +174,7 @@ func (ssp *commonSSP) BeforeSummary(event *beat.Event) BeforeSummaryActions { eventext.MergeEventFields(event, fields) - logp.L().Debugf("attempt info: current(%v) == lastStatus(%v) && attempts(%d < %d)", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts) + logp.L().Infof("attempt info: current(%v) == lastStatus(%v) && attempts(%d < %d)", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts) if retry { return RetryBeforeSummary diff --git a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go index 63de2da71a81..2a94b3e6f596 100644 --- a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go +++ b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go @@ -60,10 +60,10 @@ func TestSummarizer(t *testing.T) { { "start down, transition to up", 2, - "du", - "du", - "11", - 2, + "duu", + "duu", + "121", + 3, testURL, }, { @@ -80,7 +80,7 @@ func TestSummarizer(t *testing.T) { 2, "dddddddd", "dddddddd", - "11111111", + "12111111", 8, testURL, }, diff --git a/heartbeat/monitors/wrappers/wrappers_test.go b/heartbeat/monitors/wrappers/wrappers_test.go index 26a54f9fc36f..eb4565334342 100644 --- a/heartbeat/monitors/wrappers/wrappers_test.go +++ b/heartbeat/monitors/wrappers/wrappers_test.go @@ -43,8 +43,6 @@ import ( "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/logger" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" - "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitorstate" - "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/jobsummary" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/summarizer/summarizertesthelper" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/wraputil" "github.com/elastic/beats/v7/heartbeat/scheduler/schedule" @@ -352,132 +350,6 @@ func TestMultiJobConts(t *testing.T) { }) } -// TestRetryMultiCont is of somewhat dubious utility at the moment, -// it mostly tests that we __don't__ retry on an initial down. -// retry logic is better and more completely tested in the summarizer -// and scenario tests. -func TestRetryMultiCont(t *testing.T) { - uniqScope := isdef.ScopedIsUnique() - - expected := []struct { - monStatus string - js jobsummary.JobSummary - state monitorstate.State - }{ - { - "down", - jobsummary.JobSummary{ - Status: "down", - FinalAttempt: true, - // we expect two up since this is a lightweight - // job and all events get a monitor status - // since no errors are returned that's 2 - Up: 0, - Down: 2, - Attempt: 1, - MaxAttempts: 2, - }, - monitorstate.State{ - Status: "down", - Up: 0, - Down: 2, - Checks: 2, - }, - }, - { - "down", - jobsummary.JobSummary{ - Status: "down", - FinalAttempt: true, - Up: 0, - Down: 2, - Attempt: 2, - MaxAttempts: 2, - }, - monitorstate.State{ - Status: "down", - Up: 0, - Down: 2, - Checks: 2, - }, - }, - } - - jobErr := fmt.Errorf("down") - - makeContJob := func(t *testing.T, u string) jobs.Job { - expIdx := 0 - return func(event *beat.Event) ([]jobs.Job, error) { - eventext.MergeEventFields(event, mapstr.M{"cont": "1st"}) - u, err := url.Parse(u) - require.NoError(t, err) - eventext.MergeEventFields(event, mapstr.M{"url": wraputil.URLFields(u)}) - - return []jobs.Job{ - func(event *beat.Event) ([]jobs.Job, error) { - eventext.MergeEventFields(event, mapstr.M{"cont": "2nd"}) - eventext.MergeEventFields(event, mapstr.M{"url": wraputil.URLFields(u)}) - - expIdx++ - if expIdx >= len(expected)-1 { - expIdx = 0 - } - exp := expected[expIdx] - if exp.js.Status == "down" { - return nil, jobErr - } - - return nil, nil - }, - }, jobErr - } - } - - contJobValidator := func(u string, msg string) validator.Validator { - return lookslike.Compose( - urlValidator(t, u), - hbtestllext.MaybeHasEventType, - lookslike.MustCompile(map[string]interface{}{"cont": msg}), - lookslike.MustCompile(map[string]interface{}{ - "error": map[string]interface{}{ - "message": isdef.IsString, - "type": isdef.IsString, - }, - "monitor": map[string]interface{}{ - "id": uniqScope.IsUniqueTo(u), - "name": testMonFields.Name, - "type": testMonFields.Type, - "status": "down", - "check_group": uniqScope.IsUniqueTo(u), - }, - "state": isdef.Optional(hbtestllext.IsMonitorState), - }), - hbtestllext.MonitorTimespanValidator, - ) - } - - retryMonFields := testMonFields - retryMonFields.MaxAttempts = 2 - - for _, expected := range expected { - testCommonWrap(t, testDef{ - "multi-job-continuations-retry", - retryMonFields, - []jobs.Job{makeContJob(t, "http://foo.com")}, - []validator.Validator{ - contJobValidator("http://foo.com", "1st"), - lookslike.Compose( - contJobValidator("http://foo.com", "2nd"), - summarizertesthelper.SummaryValidator(expected.js.Up, expected.js.Down), - hbtestllext.MaybeHasDuration, - ), - }, - nil, - nil, - }) - } -} - func TestMultiJobContsCancelledEvents(t *testing.T) { uniqScope := isdef.ScopedIsUnique() From ee455fc5574747dafc26160e36562103eafb3b07 Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Fri, 13 Oct 2023 12:09:39 -0700 Subject: [PATCH 03/50] Give Auditbeat k8s Clusterrole job permissions (#36703) Update the Auditbeat Kubernetes cluster role to add read permissions on jobs/cronjobs. These permissions were added to other Cluster Role permissions previously, but was missed on auditbeat. --- deploy/kubernetes/auditbeat-kubernetes.yaml | 4 ++++ deploy/kubernetes/auditbeat/auditbeat-role.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/deploy/kubernetes/auditbeat-kubernetes.yaml b/deploy/kubernetes/auditbeat-kubernetes.yaml index 064d50114b89..6bb0b2ce4b21 100644 --- a/deploy/kubernetes/auditbeat-kubernetes.yaml +++ b/deploy/kubernetes/auditbeat-kubernetes.yaml @@ -23,6 +23,10 @@ rules: resources: - replicasets verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/deploy/kubernetes/auditbeat/auditbeat-role.yaml b/deploy/kubernetes/auditbeat/auditbeat-role.yaml index 1b17fa564152..b7942328364f 100644 --- a/deploy/kubernetes/auditbeat/auditbeat-role.yaml +++ b/deploy/kubernetes/auditbeat/auditbeat-role.yaml @@ -15,6 +15,10 @@ rules: resources: - replicasets verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role From b8a241b0703d252c3ff1ba9b417aea585f811da1 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Sat, 14 Oct 2023 12:08:01 -0400 Subject: [PATCH 04/50] chore: Update snapshot.yml (#36839) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0bf8c5252e07..3d7bbd0bacb3 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-1a1295ff-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-cb808527-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-1a1295ff-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-cb808527-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-1a1295ff-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-cb808527-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 6be0d18448bb84130cf1976b1521f45717c6b2fb Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 16 Oct 2023 09:26:19 -0400 Subject: [PATCH 05/50] [Automation] Bump Golang version to 1.20.10 (#36846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * Add changelog. --------- Co-authored-by: apmmachine Co-authored-by: Craig MacKenzie --- .go-version | 2 +- .golangci.yml | 8 ++++---- CHANGELOG.next.asciidoc | 1 + auditbeat/Dockerfile | 2 +- dev-tools/kubernetes/filebeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/heartbeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/metricbeat/Dockerfile.debug | 2 +- heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- x-pack/metricbeat/module/stan/_meta/Dockerfile | 2 +- 16 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.go-version b/.go-version index 95393fc7d4de..acdfc7930c8b 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.20.8 +1.20.10 diff --git a/.golangci.yml b/.golangci.yml index 9c480e4a5253..74d04103cedf 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -113,7 +113,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.20.8" + go: "1.20.10" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -131,19 +131,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.8" + go: "1.20.10" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.8" + go: "1.20.10" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.20.8" + go: "1.20.10" gosec: excludes: diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c4ec867eaf39..549e071d5662 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -154,6 +154,7 @@ is collected by it. *Affecting all Beats* +- Upgrade to Go 1.20.10. {pull}36846[36846] - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364] - When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183] - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188] diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 7addfc95d5ff..acd45019229b 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 RUN \ apt-get update \ diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index e83f5fa7a57d..c6a70c592907 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.8 as builder +FROM golang:1.20.10 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index 54eeeb2109e7..59860c2611f8 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.8 as builder +FROM golang:1.20.10 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index e355f7e4f4e0..92190894b742 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.8 as builder +FROM golang:1.20.10 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 7a56f7219fb0..a125998427fe 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 RUN \ apt-get update \ diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 4b7fcd5eaa4c..10bef576ac71 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.11.0 :doc-branch: main -:go-version: 1.20.8 +:go-version: 1.20.10 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 58b0691291ba..ce89ebef635c 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 RUN \ apt update \ diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 33da87702cd8..d93551eb2c38 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index a1628a0388b1..b458b082ebe3 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.20.8 AS build-env +FROM golang:1.20.10 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index ef2119e2bcdf..69592bda158f 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.20.8 +FROM golang:1.20.10 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 41b7dd5a2fc4..3830de9ecb40 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 78eb698976a8..6b8797d3cde7 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.8 +FROM golang:1.20.10 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 2ca6c91c50c5..29edcfbd6bb9 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.20.8 AS build-env +FROM golang:1.20.10 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build . From 4c34c41039be5f24d9c7f2b6f3305a6b1522c969 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 20:37:02 +0200 Subject: [PATCH 06/50] docs: Prepare Changelog for 8.10.4 (#36856) (#36865) * docs: Close changelog for 8.10.4 * Update CHANGELOG.asciidoc --------- Co-authored-by: Pierre HILBERT (cherry picked from commit ea0da2d9c02647b5f6123a57bb1e66eb6c4fa0e5) Co-authored-by: Elastic Machine --- CHANGELOG.asciidoc | 5 +++++ CHANGELOG.next.asciidoc | 3 +++ libbeat/docs/release.asciidoc | 1 + 3 files changed, 9 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1b5308195060..e97fd625570c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,11 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.10.4]] +=== Beats version 8.10.4 +https://github.com/elastic/beats/compare/v8.10.3\...v8.10.4[View commits] + + [[release-notes-8.10.3]] === Beats version 8.10.3 https://github.com/elastic/beats/compare/v8.10.2\...v8.10.3[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 549e071d5662..21e7fa23daf7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -330,6 +330,9 @@ is collected by it. + + + diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 63169edc010b..a1bb2ada147a 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From b15f989c22c50490925a6571711c81cdda7688b5 Mon Sep 17 00:00:00 2001 From: Pierre HILBERT Date: Tue, 17 Oct 2023 18:26:28 +0200 Subject: [PATCH 07/50] Fix some serverless typos (#36875) --- libbeat/esleg/eslegclient/connection.go | 2 +- libbeat/template/load_integration_test.go | 10 +++++----- libbeat/template/template.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libbeat/esleg/eslegclient/connection.go b/libbeat/esleg/eslegclient/connection.go index 322d0f08dbf1..953e68439935 100644 --- a/libbeat/esleg/eslegclient/connection.go +++ b/libbeat/esleg/eslegclient/connection.go @@ -423,7 +423,7 @@ func (conn *Connection) getVersion() error { } if versionData.Version.BuildFlavor == "serverless" { - conn.log.Info("build flavor of es is severless, marking connection as serverless") + conn.log.Info("build flavor of es is serverless, marking connection as serverless") conn.isServerless = true } else if versionData.Version.BuildFlavor == "default" { conn.isServerless = false diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index cce304798d98..b3aafad5d692 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -69,7 +69,7 @@ func newTestSetup(t *testing.T, cfg TemplateConfig) *testSetup { if err := client.Connect(); err != nil { t.Fatal(err) } - handler := &mockClientHandler{severless: false, mode: lifecycle.ILM} + handler := &mockClientHandler{serverless: false, mode: lifecycle.ILM} loader, err := NewESLoader(client, handler) require.NoError(t, err) s := testSetup{t: t, client: client, loader: loader, config: cfg} @@ -86,7 +86,7 @@ func newTestSetupWithESClient(t *testing.T, client ESClient, cfg TemplateConfig) if cfg.Name == "" { cfg.Name = fmt.Sprintf("load-test-%+v", rand.Int()) } - handler := &mockClientHandler{severless: false, mode: lifecycle.ILM} + handler := &mockClientHandler{serverless: false, mode: lifecycle.ILM} loader, err := NewESLoader(client, handler) require.NoError(t, err) return &testSetup{t: t, client: client, loader: loader, config: cfg} @@ -564,11 +564,11 @@ func getTestingElasticsearch(t eslegtest.TestLogger) *eslegclient.Connection { } type mockClientHandler struct { - severless bool - mode lifecycle.Mode + serverless bool + mode lifecycle.Mode } -func (cli *mockClientHandler) IsServerless() bool { return cli.severless } +func (cli *mockClientHandler) IsServerless() bool { return cli.serverless } func (cli *mockClientHandler) CheckEnabled() (bool, error) { return true, nil } func (cli *mockClientHandler) Mode() lifecycle.Mode { return cli.mode } func (cli *mockClientHandler) IsElasticsearch() bool { return true } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index ae67e7ec542e..5663a55c9cb5 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -317,7 +317,7 @@ func buildIdxSettings(ver version.V, userSettings mapstr.M, isServerless bool) m // deal with settings that aren't available on serverless if isServerless { - logp.L().Infof("remote instance is severless, number_of_shards and max_docvalue_fields_search will be skipped in index template") + logp.L().Infof("remote instance is serverless, number_of_shards and max_docvalue_fields_search will be skipped in index template") userSettings.Delete("number_of_shards") } else { indexSettings.Put("max_docvalue_fields_search", defaultMaxDocvalueFieldsSearch) From 7fffd0f8c9440efacb00237b82aa7406cc5998d4 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:29:35 -0700 Subject: [PATCH 08/50] Fix warning log line in serverless setup (#36867) * fix warning log line * change log line * adjust language --- libbeat/idxmgmt/lifecycle/es_client_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/idxmgmt/lifecycle/es_client_handler.go b/libbeat/idxmgmt/lifecycle/es_client_handler.go index 5bc6d70eac83..973473449b9d 100644 --- a/libbeat/idxmgmt/lifecycle/es_client_handler.go +++ b/libbeat/idxmgmt/lifecycle/es_client_handler.go @@ -95,7 +95,7 @@ func NewESClientHandler(c ESClient, info beat.Info, cfg RawConfig) (*ESClientHan // if the user has set both to different values, throw a warning, as overwrite operations will probably fail if c.IsServerless() { if cfg.TemplateName != "" && cfg.TemplateName != name { - logp.L().Warnf("policy name is %s but template name is %s; under serverless, non-default template and policy names should be the same. Updates & overwrites may not work.") + logp.L().Warnf("setup.dsl.data_stream_pattern is %s, but setup.template.name is %s; under serverless, non-default template and DSL pattern names should be the same. Additional updates & overwrites to this config will not work.", name, cfg.TemplateName) } } From d3c42ef75b93bf7b833e3ea47e4a5a5e55b55002 Mon Sep 17 00:00:00 2001 From: Andrew Gizas Date: Wed, 18 Oct 2023 09:48:57 +0300 Subject: [PATCH 09/50] Updating elastic-autodiscovery version to 0.6.4 (#36877) * Updating elastic-autodiscovery version to 0.6.4 --- CHANGELOG.next.asciidoc | 2 ++ NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 21e7fa23daf7..1fd25b842017 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,6 +10,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* - The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] +- Elastic-agent-autodiscover library updated to version 0.6.4, disabling metadata for deployment and cronjob. Pods that will be created from deployments or cronjobs will not have the extra metadata field for kubernetes.deployment or kubernetes.cronjob, respectively. {pull}36877[36877] + *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 03627af32a3d..6dcf338d609e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12268,11 +12268,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.2 +Version: v0.6.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.4/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 5deb71586d1a..77bc86125193 100644 --- a/go.mod +++ b/go.mod @@ -201,7 +201,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.2 + github.com/elastic/elastic-agent-autodiscover v0.6.4 github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.7.0 diff --git a/go.sum b/go.sum index d96c6592261b..5812b9c91dce 100644 --- a/go.sum +++ b/go.sum @@ -649,8 +649,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+PiWyZ4I4F4LqrCYgYlNc= -github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU= +github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+eXCqRnRg2SQQinxP+KsA= +github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= From aef4d4f673450d26a1c7d59751e5bfec72f0b816 Mon Sep 17 00:00:00 2001 From: Julien Lind Date: Wed, 18 Oct 2023 19:15:43 +0200 Subject: [PATCH 10/50] Update functionbeat doc according to functionbeat deprecation planning (#36871) --- x-pack/functionbeat/docs/page_header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/functionbeat/docs/page_header.html b/x-pack/functionbeat/docs/page_header.html index cec30d66bbfe..5c1aaf1ba115 100644 --- a/x-pack/functionbeat/docs/page_header.html +++ b/x-pack/functionbeat/docs/page_header.html @@ -1,3 +1,3 @@ -Functionbeat will reach End of Support on October 18, 2023. You should consider +Functionbeat reached End of Support on October 18, 2023. You must consider moving your deployments to the more versatile and efficient Elastic Serverless Forwarder. From e44158767be6aa0136e738197e8180c1fb3706e1 Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:25:01 -0400 Subject: [PATCH 11/50] Update default loadbalance setting for Logstash to 'false' (#36895) --- libbeat/outputs/logstash/docs/logstash.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/outputs/logstash/docs/logstash.asciidoc b/libbeat/outputs/logstash/docs/logstash.asciidoc index 8ab48e325a47..28a7df42dda4 100644 --- a/libbeat/outputs/logstash/docs/logstash.asciidoc +++ b/libbeat/outputs/logstash/docs/logstash.asciidoc @@ -286,7 +286,7 @@ list of configured hosts over time, use this option in conjunction with the `ttl` setting to close the connection at the configured interval and choose a new target host. -The default value is `true`. +The default value is `false`. ["source","yaml",subs="attributes"] ------------------------------------------------------------------------------ From deb7d4268b4c70881a07e5adf093c46315c519c1 Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:14:46 -0500 Subject: [PATCH 12/50] upgrade elastic-agent-libs to v0.6.0 (#36896) * upgrade elastic-agent-libs to v0.6.0 allows beat running as a windows service to receive more than one change request. --- CHANGELOG.next.asciidoc | 2 +- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1fd25b842017..37bb60d04e3f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -68,7 +68,7 @@ is collected by it. - Add default cgroup regex for add_process_metadata processor {pull}36484[36484] {issue}32961[32961] - Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471] - syslog processor - Fix the ability to use `when` conditions on the processor. {issue}36762[36762] - +- upgrade elastic-agent-libs to v0.6.0, allows beat running as a windows service to receive more than one change request. {pull}36896[36896] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 6dcf338d609e..6ae9f58daba1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12712,11 +12712,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.3.15-0.20230913212237-dbdaf18c898b +Version: v0.6.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.15-0.20230913212237-dbdaf18c898b/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 77bc86125193..e4955abcc912 100644 --- a/go.mod +++ b/go.mod @@ -202,7 +202,7 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.6.4 - github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b + github.com/elastic/elastic-agent-libs v0.6.0 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.7.0 github.com/elastic/go-elasticsearch/v8 v8.10.0 diff --git a/go.sum b/go.sum index 5812b9c91dce..a1aecdb59dfd 100644 --- a/go.sum +++ b/go.sum @@ -653,8 +653,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+e github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b h1:a2iuOokwld+D7VhyFymVtsPoqxZ8fkkOCOOjeYU9CDM= -github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ= +github.com/elastic/elastic-agent-libs v0.6.0 h1:HnL/OpAzIHlK8y1J69XQuAx4tlCzd6e2kldMHvXARvY= +github.com/elastic/elastic-agent-libs v0.6.0/go.mod h1:K6U+n84siZ66ZyG36h1/x+fw1oIZbFXEypAC6KSiFOg= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= From e39b37e02b029f7bb8d6eff01157efe4994e5d13 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:45:08 -0700 Subject: [PATCH 13/50] Default `allow_older_versions` to true (#36884) * default allow_older_versions to true * fix removed file * fix error message * Changing tests according to behavior change * update docs * add changelog --------- Co-authored-by: Pierre HILBERT --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 5 ++--- filebeat/filebeat.reference.yml | 5 ++--- heartbeat/heartbeat.reference.yml | 5 ++--- .../config/output-elasticsearch.reference.yml.tmpl | 5 ++--- libbeat/cmd/instance/beat.go | 10 ++++++---- libbeat/cmd/instance/beat_test.go | 6 +++--- metricbeat/metricbeat.reference.yml | 5 ++--- packetbeat/packetbeat.reference.yml | 5 ++--- winlogbeat/winlogbeat.reference.yml | 5 ++--- x-pack/auditbeat/auditbeat.reference.yml | 5 ++--- x-pack/filebeat/filebeat.reference.yml | 5 ++--- x-pack/functionbeat/functionbeat.reference.yml | 5 ++--- x-pack/heartbeat/heartbeat.reference.yml | 5 ++--- x-pack/metricbeat/metricbeat.reference.yml | 5 ++--- x-pack/osquerybeat/osquerybeat.reference.yml | 5 ++--- x-pack/packetbeat/packetbeat.reference.yml | 5 ++--- x-pack/winlogbeat/winlogbeat.reference.yml | 5 ++--- 18 files changed, 40 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 37bb60d04e3f..be6b54838e78 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -166,6 +166,7 @@ is collected by it. - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322] - [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor +- Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 32fbef2da046..d4214eaf604b 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -525,9 +525,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # auditbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents auditbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 7f86b5aa9d2d..45aff60ce236 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1621,9 +1621,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # filebeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents filebeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 89fc08ef8e66..e8b74f8c075e 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -617,9 +617,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # heartbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents heartbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 48f1ba2c007b..4acd341da01e 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -84,9 +84,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # {{.BeatName}} expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents {{.BeatName}} from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true {{include "ssl.reference.yml.tmpl" . | indent 2 }} diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 4e72996c9660..e6596106b83c 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -1028,16 +1028,18 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { func (b *Beat) registerESVersionCheckCallback() error { _, err := elasticsearch.RegisterGlobalCallback(func(conn *eslegclient.Connection) error { if !isElasticsearchOutput(b.Config.Output.Name()) { - return errors.New("Elasticsearch output is not configured") + return errors.New("elasticsearch output is not configured") } - if b.isConnectionToOlderVersionAllowed() { + // if we allow older versions, return early and don't check versions + // versions don't matter on serverless, so always bypass + if b.isConnectionToOlderVersionAllowed() || conn.IsServerless() { return nil } esVersion := conn.GetVersion() beatVersion, err := libversion.New(b.Info.Version) if err != nil { - return err + return fmt.Errorf("error fetching version from elasticsearch: %w", err) } if esVersion.LessThanMajorMinor(beatVersion) { return fmt.Errorf("%w ES=%s, Beat=%s", elasticsearch.ErrTooOld, esVersion.String(), b.Info.Version) @@ -1051,7 +1053,7 @@ func (b *Beat) registerESVersionCheckCallback() error { func (b *Beat) isConnectionToOlderVersionAllowed() bool { config := struct { AllowOlder bool `config:"allow_older_versions"` - }{false} + }{true} _ = b.Config.Output.Config().Unpack(&config) diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index b6834d89b5d8..fc8c88a49159 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -236,7 +236,7 @@ func TestReloader(t *testing.T) { elasticsearch: hosts: ["https://127.0.0.1:9200"] username: "elastic" - allow_older_versions: true + allow_older_versions: false ` c, err := config.NewConfigWithYAML([]byte(cfg), cfg) require.NoError(t, err) @@ -248,13 +248,13 @@ elasticsearch: reloader := b.makeOutputReloader(m) require.False(t, b.Config.Output.IsSet(), "the output should not be set yet") - require.False(t, b.isConnectionToOlderVersionAllowed(), "the flag should not be present in the empty configuration") + require.True(t, b.isConnectionToOlderVersionAllowed(), "allow_older_versions flag should be true from 8.11") err = reloader.Reload(update) require.NoError(t, err) require.True(t, b.Config.Output.IsSet(), "now the output should be set") require.Equal(t, outCfg, b.Config.Output.Config()) require.Same(t, c, m.cfg.Config) - require.True(t, b.isConnectionToOlderVersionAllowed(), "the flag should be present") + require.False(t, b.isConnectionToOlderVersionAllowed(), "allow_older_versions flag should now be set to false") }) } diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 88e353e883b7..fc79ddb514c9 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1360,9 +1360,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # metricbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents metricbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index aebeb1947a63..cc05f7b52128 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -991,9 +991,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # packetbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents packetbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 5619e9a6375d..110370957cf7 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -407,9 +407,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # winlogbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents winlogbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 999f0416354e..6d9a71ca99cc 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -581,9 +581,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # auditbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents auditbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 29704a80ad4e..bff96ef19973 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3991,9 +3991,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # filebeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents filebeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index b9a54bd88ec5..d3a2231a43ef 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -649,9 +649,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # functionbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents functionbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 89fc08ef8e66..e8b74f8c075e 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -617,9 +617,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # heartbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents heartbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 63acc6f52453..436693bdfbc7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1921,9 +1921,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # metricbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents metricbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index d6770083e628..f17d16e28b8a 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -368,9 +368,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # osquerybeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents osquerybeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index aebeb1947a63..cc05f7b52128 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -991,9 +991,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # packetbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents packetbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 5d52a07b37bc..eec0bca80779 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -409,9 +409,8 @@ output.elasticsearch: # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 - # winlogbeat expects Elasticsearch to be the same version or newer than the Beat. - # Lift the version restriction by setting allow_older_versions to true. - #allow_older_versions: false + # Prevents winlogbeat from connecting to older Elasticsearch versions when set to `false` + #allow_older_versions: true # Use SSL settings for HTTPS. #ssl.enabled: true From 0bd2d7392b3e1d2cf3340f67c5f390c74857819d Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 19 Oct 2023 18:04:30 -0500 Subject: [PATCH 14/50] add support for queue settings under outputs (#36788) * add support for queue settings under outputs --- CHANGELOG.next.asciidoc | 4 +- libbeat/cmd/instance/beat.go | 49 ++++++ libbeat/cmd/instance/beat_test.go | 164 ++++++++++++++++++ libbeat/docs/queueconfig.asciidoc | 8 +- libbeat/outputs/console/config.go | 6 +- libbeat/outputs/console/console.go | 14 +- libbeat/outputs/elasticsearch/config.go | 1 + .../outputs/elasticsearch/elasticsearch.go | 34 ++-- libbeat/outputs/fileout/config.go | 24 +-- libbeat/outputs/fileout/file.go | 10 +- libbeat/outputs/kafka/config.go | 7 +- libbeat/outputs/kafka/kafka.go | 12 +- libbeat/outputs/logstash/config.go | 1 + libbeat/outputs/logstash/logstash.go | 18 +- libbeat/outputs/redis/config.go | 2 + libbeat/outputs/redis/redis.go | 34 ++-- libbeat/outputs/util.go | 57 +++++- libbeat/publisher/pipeline/client_test.go | 2 +- libbeat/publisher/pipeline/controller.go | 13 +- libbeat/publisher/pipeline/controller_test.go | 2 +- libbeat/publisher/pipeline/pipeline.go | 10 +- libbeat/publisher/pipeline/stress/out.go | 13 +- libbeat/publisher/queue/diskqueue/queue.go | 1 + libbeat/publisher/queue/memqueue/broker.go | 7 +- .../publisher/queue/memqueue/queue_test.go | 12 +- libbeat/publisher/queue/proxy/broker.go | 1 + libbeat/publisher/queue/queue.go | 2 +- 27 files changed, 383 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be6b54838e78..7c35197567b8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -166,6 +166,7 @@ is collected by it. - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322] - [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor +- allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788] - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] *Auditbeat* @@ -216,8 +217,7 @@ is collected by it. - Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273] - Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938] - Add device handling to Okta input package for entity analytics. {pull}36049[36049] -- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] -- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}36286[36286] +- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30916[30916] {pull}36286[36286] - [Azure] Add input metrics to the azure-eventhub input. {pull}35739[35739] - Reduce HTTPJSON metrics allocations. {pull}36282[36282] - Add support for a simplified input configuraton when running under Elastic-Agent {pull}36390[36390] diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index e6596106b83c..efe8bd48f79a 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -66,6 +66,7 @@ import ( "github.com/elastic/beats/v7/libbeat/pprof" "github.com/elastic/beats/v7/libbeat/publisher/pipeline" "github.com/elastic/beats/v7/libbeat/publisher/processing" + "github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue" "github.com/elastic/beats/v7/libbeat/version" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/file" @@ -783,6 +784,10 @@ func (b *Beat) configure(settings Settings) error { return fmt.Errorf("error unpacking config data: %w", err) } + if err := promoteOutputQueueSettings(&b.Config); err != nil { + return fmt.Errorf("could not promote output queue settings: %w", err) + } + if err := features.UpdateFromConfig(b.RawConfig); err != nil { return fmt.Errorf("could not parse features: %w", err) } @@ -1482,3 +1487,47 @@ func sanitizeIPs(ips []string) []string { } return validIPs } + +// promoteOutputQueueSettings checks to see if the output +// configuration has queue settings defined and if so it promotes them +// to the top level queue settings. This is done to allow existing +// behavior of specifying queue settings at the top level or like +// elastic-agent that specifies queue settings under the output +func promoteOutputQueueSettings(bc *beatConfig) error { + if bc.Output.IsSet() && bc.Output.Config().Enabled() { + pc := pipeline.Config{} + err := bc.Output.Config().Unpack(&pc) + if err != nil { + return fmt.Errorf("error unpacking output queue settings: %w", err) + } + if pc.Queue.IsSet() { + logp.Info("global queue settings replaced with output queue settings") + bc.Pipeline.Queue = pc.Queue + } + } + return nil +} + +func (bc *beatConfig) Validate() error { + if bc.Output.IsSet() && bc.Output.Config().Enabled() { + outputPC := pipeline.Config{} + err := bc.Output.Config().Unpack(&outputPC) + if err != nil { + return fmt.Errorf("error unpacking output queue settings: %w", err) + } + if bc.Pipeline.Queue.IsSet() && outputPC.Queue.IsSet() { + return fmt.Errorf("top level queue and output level queue settings defined, only one is allowed") + } + //elastic-agent doesn't support disk queue yet + if bc.Management.Enabled() && outputPC.Queue.Config().Enabled() && outputPC.Queue.Name() == diskqueue.QueueType { + return fmt.Errorf("disk queue is not supported when management is enabled") + } + } + + //elastic-agent doesn't support disk queue yet + if bc.Management.Enabled() && bc.Pipeline.Queue.Config().Enabled() && bc.Pipeline.Queue.Name() == diskqueue.QueueType { + return fmt.Errorf("disk queue is not supported when management is enabled") + } + + return nil +} diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index fc8c88a49159..03474ecfcd91 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -27,7 +27,9 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/outputs" + "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-ucfg/yaml" "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" @@ -269,3 +271,165 @@ func (r *outputReloaderMock) Reload( r.cfg = cfg return nil } + +func TestPromoteOutputQueueSettings(t *testing.T) { + tests := map[string]struct { + input []byte + memEvents int + }{ + "blank": { + input: []byte(""), + memEvents: 4096, + }, + "defaults": { + input: []byte(` +name: mockbeat +output: + elasticsearch: + hosts: + - "localhost:9200" +`), + memEvents: 4096, + }, + "topLevelQueue": { + input: []byte(` +name: mockbeat +queue: + mem: + events: 8096 +output: + elasticsearch: + hosts: + - "localhost:9200" +`), + memEvents: 8096, + }, + "outputLevelQueue": { + input: []byte(` +name: mockbeat +output: + elasticsearch: + hosts: + - "localhost:9200" + queue: + mem: + events: 8096 +`), + memEvents: 8096, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := yaml.NewConfig(tc.input) + require.NoError(t, err) + + config := beatConfig{} + err = cfg.Unpack(&config) + require.NoError(t, err) + + err = promoteOutputQueueSettings(&config) + require.NoError(t, err) + + ms, err := memqueue.SettingsForUserConfig(config.Pipeline.Queue.Config()) + require.NoError(t, err) + require.Equalf(t, tc.memEvents, ms.Events, "config was: %v", config.Pipeline.Queue.Config()) + }) + } +} + +func TestValidateBeatConfig(t *testing.T) { + tests := map[string]struct { + input []byte + expectValidationError string + }{ + "blank": { + input: []byte(""), + expectValidationError: "", + }, + "defaults": { + input: []byte(` +name: mockbeat +output: + elasticsearch: + hosts: + - "localhost:9200" +`), + expectValidationError: "", + }, + "topAndOutputLevelQueue": { + input: []byte(` +name: mockbeat +queue: + mem: + events: 2048 +output: + elasticsearch: + hosts: + - "localhost:9200" + queue: + mem: + events: 8096 +`), + expectValidationError: "top level queue and output level queue settings defined, only one is allowed accessing config", + }, + "managementTopLevelDiskQueue": { + input: []byte(` +name: mockbeat +management: + enabled: true +queue: + disk: + max_size: 1G +output: + elasticsearch: + hosts: + - "localhost:9200" +`), + expectValidationError: "disk queue is not supported when management is enabled accessing config", + }, + "managementOutputLevelDiskQueue": { + input: []byte(` +name: mockbeat +management: + enabled: true +output: + elasticsearch: + hosts: + - "localhost:9200" + queue: + disk: + max_size: 1G +`), + expectValidationError: "disk queue is not supported when management is enabled accessing config", + }, + "managementFalseOutputLevelDiskQueue": { + input: []byte(` +name: mockbeat +management: + enabled: false +output: + elasticsearch: + hosts: + - "localhost:9200" + queue: + disk: + max_size: 1G +`), + expectValidationError: "", + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := yaml.NewConfig(tc.input) + require.NoError(t, err) + config := beatConfig{} + err = cfg.Unpack(&config) + if tc.expectValidationError != "" { + require.Error(t, err) + require.Equal(t, tc.expectValidationError, err.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc index fb930831dac3..ade3bd2ec8ed 100644 --- a/libbeat/docs/queueconfig.asciidoc +++ b/libbeat/docs/queueconfig.asciidoc @@ -9,10 +9,10 @@ queue is responsible for buffering and combining events into batches that can be consumed by the outputs. The outputs will use bulk operations to send a batch of events in one transaction. -You can configure the type and behavior of the internal queue by setting -options in the `queue` section of the +{beatname_lc}.yml+ config file. Only one -queue type can be configured. - +You can configure the type and behavior of the internal queue by +setting options in the `queue` section of the +{beatname_lc}.yml+ +config file or by setting options in the `queue` section of the +output. Only one queue type can be configured. This sample configuration sets the memory queue to buffer up to 4096 events: diff --git a/libbeat/outputs/console/config.go b/libbeat/outputs/console/config.go index 44869e388fa9..e0a1cc9ff280 100644 --- a/libbeat/outputs/console/config.go +++ b/libbeat/outputs/console/config.go @@ -17,7 +17,10 @@ package console -import "github.com/elastic/beats/v7/libbeat/outputs/codec" +import ( + "github.com/elastic/beats/v7/libbeat/outputs/codec" + "github.com/elastic/elastic-agent-libs/config" +) type Config struct { Codec codec.Config `config:"codec"` @@ -26,6 +29,7 @@ type Config struct { Pretty bool `config:"pretty"` BatchSize int + Queue config.Namespace `config:"queue"` } var defaultConfig = Config{} diff --git a/libbeat/outputs/console/console.go b/libbeat/outputs/console/console.go index 905aa778998c..b81bf3363486 100644 --- a/libbeat/outputs/console/console.go +++ b/libbeat/outputs/console/console.go @@ -23,7 +23,6 @@ import ( "fmt" "os" "runtime" - "time" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/outputs" @@ -43,13 +42,6 @@ type console struct { index string } -type consoleEvent struct { - Timestamp time.Time `json:"@timestamp" struct:"@timestamp"` - - // Note: stdlib json doesn't support inlining :( -> use `codec: 2`, to generate proper event - Fields interface{} `struct:",inline"` -} - func init() { outputs.RegisterType("console", makeConsole) } @@ -82,18 +74,18 @@ func makeConsole( index := beat.Beat c, err := newConsole(index, observer, enc) if err != nil { - return outputs.Fail(fmt.Errorf("console output initialization failed with: %v", err)) + return outputs.Fail(fmt.Errorf("console output initialization failed with: %w", err)) } // check stdout actually being available if runtime.GOOS != "windows" { if _, err = c.out.Stat(); err != nil { - err = fmt.Errorf("console output initialization failed with: %v", err) + err = fmt.Errorf("console output initialization failed with: %w", err) return outputs.Fail(err) } } - return outputs.Success(config.BatchSize, 0, c) + return outputs.Success(config.Queue, config.BatchSize, 0, c) } func newConsole(index string, observer outputs.Observer, codec codec.Codec) (*console, error) { diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index ca77a44b8331..e504f2dc213c 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -45,6 +45,7 @@ type elasticsearchConfig struct { AllowOlderVersion bool `config:"allow_older_versions"` Transport httpcommon.HTTPTransportSettings `config:",inline"` + Queue config.Namespace `config:"queue"` } type Backoff struct { diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 9cd33ea8d8a1..f7e388539243 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -41,7 +41,7 @@ func makeES( ) (outputs.Group, error) { log := logp.NewLogger(logSelector) if !cfg.HasField("bulk_max_size") { - cfg.SetInt("bulk_max_size", -1, defaultBulkSize) + _ = cfg.SetInt("bulk_max_size", -1, defaultBulkSize) } index, pipeline, err := buildSelectors(im, beat, cfg) @@ -49,12 +49,12 @@ func makeES( return outputs.Fail(err) } - config := defaultConfig - if err := cfg.Unpack(&config); err != nil { + esConfig := defaultConfig + if err := cfg.Unpack(&esConfig); err != nil { return outputs.Fail(err) } - policy, err := newNonIndexablePolicy(config.NonIndexablePolicy) + policy, err := newNonIndexablePolicy(esConfig.NonIndexablePolicy) if err != nil { log.Errorf("error while creating file identifier: %v", err) return outputs.Fail(err) @@ -65,12 +65,12 @@ func makeES( return outputs.Fail(err) } - if proxyURL := config.Transport.Proxy.URL; proxyURL != nil && !config.Transport.Proxy.Disable { + if proxyURL := esConfig.Transport.Proxy.URL; proxyURL != nil && !esConfig.Transport.Proxy.Disable { log.Debugf("breaking down proxy URL. Scheme: '%s', host[:port]: '%s', path: '%s'", proxyURL.Scheme, proxyURL.Host, proxyURL.Path) log.Infof("Using proxy URL: %s", proxyURL) } - params := config.Params + params := esConfig.Params if len(params) == 0 { params = nil } @@ -84,7 +84,7 @@ func makeES( clients := make([]outputs.NetworkClient, len(hosts)) for i, host := range hosts { - esURL, err := common.MakeURL(config.Protocol, config.Path, host, 9200) + esURL, err := common.MakeURL(esConfig.Protocol, esConfig.Path, host, 9200) if err != nil { log.Errorf("Invalid host param set: %s, Error: %+v", host, err) return outputs.Fail(err) @@ -95,16 +95,16 @@ func makeES( ConnectionSettings: eslegclient.ConnectionSettings{ URL: esURL, Beatname: beat.Beat, - Kerberos: config.Kerberos, - Username: config.Username, - Password: config.Password, - APIKey: config.APIKey, + Kerberos: esConfig.Kerberos, + Username: esConfig.Username, + Password: esConfig.Password, + APIKey: esConfig.APIKey, Parameters: params, - Headers: config.Headers, - CompressionLevel: config.CompressionLevel, + Headers: esConfig.Headers, + CompressionLevel: esConfig.CompressionLevel, Observer: observer, - EscapeHTML: config.EscapeHTML, - Transport: config.Transport, + EscapeHTML: esConfig.EscapeHTML, + Transport: esConfig.Transport, }, Index: index, Pipeline: pipeline, @@ -115,11 +115,11 @@ func makeES( return outputs.Fail(err) } - client = outputs.WithBackoff(client, config.Backoff.Init, config.Backoff.Max) + client = outputs.WithBackoff(client, esConfig.Backoff.Init, esConfig.Backoff.Max) clients[i] = client } - return outputs.SuccessNet(config.LoadBalance, config.BulkMaxSize, config.MaxRetries, clients) + return outputs.SuccessNet(esConfig.Queue, esConfig.LoadBalance, esConfig.BulkMaxSize, esConfig.MaxRetries, clients) } func buildSelectors( diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index cfd28bfaaf26..e72a9f87d6fc 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -21,21 +21,23 @@ import ( "fmt" "github.com/elastic/beats/v7/libbeat/outputs/codec" + "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/file" ) -type config struct { - Path string `config:"path"` - Filename string `config:"filename"` - RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` - NumberOfFiles uint `config:"number_of_files"` - Codec codec.Config `config:"codec"` - Permissions uint32 `config:"permissions"` - RotateOnStartup bool `config:"rotate_on_startup"` +type fileOutConfig struct { + Path string `config:"path"` + Filename string `config:"filename"` + RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` + NumberOfFiles uint `config:"number_of_files"` + Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` + RotateOnStartup bool `config:"rotate_on_startup"` + Queue config.Namespace `config:"queue"` } -func defaultConfig() config { - return config{ +func defaultConfig() fileOutConfig { + return fileOutConfig{ NumberOfFiles: 7, RotateEveryKb: 10 * 1024, Permissions: 0600, @@ -43,7 +45,7 @@ func defaultConfig() config { } } -func (c *config) Validate() error { +func (c *fileOutConfig) Validate() error { if c.NumberOfFiles < 2 || c.NumberOfFiles > file.MaxBackupsLimit { return fmt.Errorf("the number_of_files to keep should be between 2 and %v", file.MaxBackupsLimit) diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 949d835f5419..d12a11b25c3c 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -51,8 +51,8 @@ func makeFileout( observer outputs.Observer, cfg *c.C, ) (outputs.Group, error) { - config := defaultConfig() - if err := cfg.Unpack(&config); err != nil { + foConfig := defaultConfig() + if err := cfg.Unpack(&foConfig); err != nil { return outputs.Fail(err) } @@ -64,14 +64,14 @@ func makeFileout( beat: beat, observer: observer, } - if err := fo.init(beat, config); err != nil { + if err := fo.init(beat, foConfig); err != nil { return outputs.Fail(err) } - return outputs.Success(-1, 0, fo) + return outputs.Success(foConfig.Queue, -1, 0, fo) } -func (out *fileOutput) init(beat beat.Info, c config) error { +func (out *fileOutput) init(beat beat.Info, c fileOutConfig) error { var path string if c.Filename != "" { path = filepath.Join(c.Path, c.Filename) diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 7247699500f5..8fff8dad0d5c 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -76,6 +76,7 @@ type kafkaConfig struct { Codec codec.Config `config:"codec"` Sasl kafka.SaslConfig `config:"sasl"` EnableFAST bool `config:"enable_krb5_fast"` + Queue config.Namespace `config:"queue"` } type metaConfig struct { @@ -101,12 +102,6 @@ var compressionModes = map[string]sarama.CompressionCodec{ "snappy": sarama.CompressionSnappy, } -const ( - saslTypePlaintext = sarama.SASLTypePlaintext - saslTypeSCRAMSHA256 = sarama.SASLTypeSCRAMSHA256 - saslTypeSCRAMSHA512 = sarama.SASLTypeSCRAMSHA512 -) - func defaultConfig() kafkaConfig { return kafkaConfig{ Hosts: nil, diff --git a/libbeat/outputs/kafka/kafka.go b/libbeat/outputs/kafka/kafka.go index ef1c253981f3..0c856ea425db 100644 --- a/libbeat/outputs/kafka/kafka.go +++ b/libbeat/outputs/kafka/kafka.go @@ -47,7 +47,7 @@ func makeKafka( log := logp.NewLogger(logSelector) log.Debug("initialize kafka output") - config, err := readConfig(cfg) + kConfig, err := readConfig(cfg) if err != nil { return outputs.Fail(err) } @@ -57,7 +57,7 @@ func makeKafka( return outputs.Fail(err) } - libCfg, err := newSaramaConfig(log, config) + libCfg, err := newSaramaConfig(log, kConfig) if err != nil { return outputs.Fail(err) } @@ -67,21 +67,21 @@ func makeKafka( return outputs.Fail(err) } - codec, err := codec.CreateEncoder(beat, config.Codec) + codec, err := codec.CreateEncoder(beat, kConfig.Codec) if err != nil { return outputs.Fail(err) } - client, err := newKafkaClient(observer, hosts, beat.IndexPrefix, config.Key, topic, config.Headers, codec, libCfg) + client, err := newKafkaClient(observer, hosts, beat.IndexPrefix, kConfig.Key, topic, kConfig.Headers, codec, libCfg) if err != nil { return outputs.Fail(err) } retry := 0 - if config.MaxRetries < 0 { + if kConfig.MaxRetries < 0 { retry = -1 } - return outputs.Success(config.BulkMaxSize, retry, client) + return outputs.Success(kConfig.Queue, kConfig.BulkMaxSize, retry, client) } func buildTopicSelector(cfg *config.C) (outil.Selector, error) { diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 82747fe01d09..9df57514495b 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -43,6 +43,7 @@ type Config struct { Proxy transport.ProxyConfig `config:",inline"` Backoff Backoff `config:"backoff"` EscapeHTML bool `config:"escape_html"` + Queue config.Namespace `config:"queue"` } type Backoff struct { diff --git a/libbeat/outputs/logstash/logstash.go b/libbeat/outputs/logstash/logstash.go index 5e7cdfeee7a5..072ec049f6fb 100644 --- a/libbeat/outputs/logstash/logstash.go +++ b/libbeat/outputs/logstash/logstash.go @@ -41,7 +41,7 @@ func makeLogstash( observer outputs.Observer, cfg *conf.C, ) (outputs.Group, error) { - config, err := readConfig(cfg, beat) + lsConfig, err := readConfig(cfg, beat) if err != nil { return outputs.Fail(err) } @@ -51,14 +51,14 @@ func makeLogstash( return outputs.Fail(err) } - tls, err := tlscommon.LoadTLSConfig(config.TLS) + tls, err := tlscommon.LoadTLSConfig(lsConfig.TLS) if err != nil { return outputs.Fail(err) } transp := transport.Config{ - Timeout: config.Timeout, - Proxy: &config.Proxy, + Timeout: lsConfig.Timeout, + Proxy: &lsConfig.Proxy, TLS: tls, Stats: observer, } @@ -72,18 +72,18 @@ func makeLogstash( return outputs.Fail(err) } - if config.Pipelining > 0 { - client, err = newAsyncClient(beat, conn, observer, config) + if lsConfig.Pipelining > 0 { + client, err = newAsyncClient(beat, conn, observer, lsConfig) } else { - client, err = newSyncClient(beat, conn, observer, config) + client, err = newSyncClient(beat, conn, observer, lsConfig) } if err != nil { return outputs.Fail(err) } - client = outputs.WithBackoff(client, config.Backoff.Init, config.Backoff.Max) + client = outputs.WithBackoff(client, lsConfig.Backoff.Init, lsConfig.Backoff.Max) clients[i] = client } - return outputs.SuccessNet(config.LoadBalance, config.BulkMaxSize, config.MaxRetries, clients) + return outputs.SuccessNet(lsConfig.Queue, lsConfig.LoadBalance, lsConfig.BulkMaxSize, lsConfig.MaxRetries, clients) } diff --git a/libbeat/outputs/redis/config.go b/libbeat/outputs/redis/config.go index 01c8f2e0238b..4785af137f10 100644 --- a/libbeat/outputs/redis/config.go +++ b/libbeat/outputs/redis/config.go @@ -22,6 +22,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/outputs/codec" + "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/transport" "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) @@ -40,6 +41,7 @@ type redisConfig struct { Db int `config:"db"` DataType string `config:"datatype"` Backoff backoff `config:"backoff"` + Queue config.Namespace `config:"queue"` } type backoff struct { diff --git a/libbeat/outputs/redis/redis.go b/libbeat/outputs/redis/redis.go index 026cb04d4f8e..9814d6abee7b 100644 --- a/libbeat/outputs/redis/redis.go +++ b/libbeat/outputs/redis/redis.go @@ -34,10 +34,6 @@ import ( "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) -type redisOut struct { - beat beat.Info -} - const ( defaultWaitRetry = 1 * time.Second defaultMaxWaitRetry = 60 * time.Second @@ -58,7 +54,9 @@ func makeRedis( ) (outputs.Group, error) { if !cfg.HasField("index") { - cfg.SetString("index", -1, beat.Beat) + if err := cfg.SetString("index", -1, beat.Beat); err != nil { + return outputs.Fail(err) + } } err := cfgwarn.CheckRemoved6xSettings(cfg, "port") @@ -77,13 +75,13 @@ func makeRedis( } } - config := defaultConfig - if err := cfg.Unpack(&config); err != nil { + rConfig := defaultConfig + if err := cfg.Unpack(&rConfig); err != nil { return outputs.Fail(err) } var dataType redisDataType - switch config.DataType { + switch rConfig.DataType { case "", "list": dataType = redisListType case "channel": @@ -102,7 +100,7 @@ func makeRedis( return outputs.Fail(err) } - tls, err := tlscommon.LoadTLSConfig(config.TLS) + tls, err := tlscommon.LoadTLSConfig(rConfig.TLS) if err != nil { return outputs.Fail(err) } @@ -129,8 +127,8 @@ func makeRedis( } transp := transport.Config{ - Timeout: config.Timeout, - Proxy: &config.Proxy, + Timeout: rConfig.Timeout, + Proxy: &rConfig.Proxy, TLS: tls, Stats: observer, } @@ -138,7 +136,7 @@ func makeRedis( switch hostUrl.Scheme { case redisScheme: if hasScheme { - transp.TLS = nil // disable TLS if user explicitely set `redis` scheme + transp.TLS = nil // disable TLS if user explicitly set `redis` scheme } case tlsRedisScheme: if transp.TLS == nil { @@ -151,23 +149,23 @@ func makeRedis( return outputs.Fail(err) } - pass := config.Password + pass := rConfig.Password hostPass, passSet := hostUrl.User.Password() if passSet { pass = hostPass } - enc, err := codec.CreateEncoder(beat, config.Codec) + enc, err := codec.CreateEncoder(beat, rConfig.Codec) if err != nil { return outputs.Fail(err) } - client := newClient(conn, observer, config.Timeout, - pass, config.Db, key, dataType, config.Index, enc) - clients[i] = newBackoffClient(client, config.Backoff.Init, config.Backoff.Max) + client := newClient(conn, observer, rConfig.Timeout, + pass, rConfig.Db, key, dataType, rConfig.Index, enc) + clients[i] = newBackoffClient(client, rConfig.Backoff.Init, rConfig.Backoff.Max) } - return outputs.SuccessNet(config.LoadBalance, config.BulkMaxSize, config.MaxRetries, clients) + return outputs.SuccessNet(rConfig.Queue, rConfig.LoadBalance, rConfig.BulkMaxSize, rConfig.MaxRetries, clients) } func buildKeySelector(cfg *config.C) (outil.Selector, error) { diff --git a/libbeat/outputs/util.go b/libbeat/outputs/util.go index 15068910f8c6..ce8765b5c2e9 100644 --- a/libbeat/outputs/util.go +++ b/libbeat/outputs/util.go @@ -17,16 +17,52 @@ package outputs +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/publisher" + "github.com/elastic/beats/v7/libbeat/publisher/queue" + "github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue" + "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" + "github.com/elastic/elastic-agent-libs/config" +) + // Fail helper can be used by output factories, to create a failure response when // loading an output must return an error. func Fail(err error) (Group, error) { return Group{}, err } -// Success create a valid output Group response for a set of client instances. -func Success(batchSize, retry int, clients ...Client) (Group, error) { +// Success create a valid output Group response for a set of client +// instances. The first argument is expected to contain a queue +// config.Namespace. The queue config is passed to assign the queue +// factory when elastic-agent reloads the output. +func Success(cfg config.Namespace, batchSize, retry int, clients ...Client) (Group, error) { + var q queue.QueueFactory + if cfg.IsSet() && cfg.Config().Enabled() { + switch cfg.Name() { + case memqueue.QueueType: + settings, err := memqueue.SettingsForUserConfig(cfg.Config()) + if err != nil { + return Group{}, fmt.Errorf("unable to get memory queue settings: %w", err) + } + q = memqueue.FactoryForSettings(settings) + case diskqueue.QueueType: + if publisher.UnderAgent() { + return Group{}, fmt.Errorf("disk queue not supported under agent") + } + settings, err := diskqueue.SettingsForUserConfig(cfg.Config()) + if err != nil { + return Group{}, fmt.Errorf("unable to get disk queue settings: %w", err) + } + q = diskqueue.FactoryForSettings(settings) + default: + return Group{}, fmt.Errorf("unknown queue type: %s", cfg.Name()) + } + } return Group{ - Clients: clients, - BatchSize: batchSize, - Retry: retry, + Clients: clients, + BatchSize: batchSize, + Retry: retry, + QueueFactory: q, }, nil } @@ -39,11 +75,16 @@ func NetworkClients(netclients []NetworkClient) []Client { return clients } -func SuccessNet(loadbalance bool, batchSize, retry int, netclients []NetworkClient) (Group, error) { +// SuccessNet create a valid output Group and creates client instances +// The first argument is expected to contain a queue config.Namespace. +// The queue config is passed to assign the queue factory when +// elastic-agent reloads the output. +func SuccessNet(cfg config.Namespace, loadbalance bool, batchSize, retry int, netclients []NetworkClient) (Group, error) { + if !loadbalance { - return Success(batchSize, retry, NewFailoverClient(netclients)) + return Success(cfg, batchSize, retry, NewFailoverClient(netclients)) } clients := NetworkClients(netclients) - return Success(batchSize, retry, clients...) + return Success(cfg, batchSize, retry, clients...) } diff --git a/libbeat/publisher/pipeline/client_test.go b/libbeat/publisher/pipeline/client_test.go index 4a212092c7e7..15260172ff54 100644 --- a/libbeat/publisher/pipeline/client_test.go +++ b/libbeat/publisher/pipeline/client_test.go @@ -144,7 +144,7 @@ func TestClientWaitClose(t *testing.T) { err := logp.TestingSetup() assert.Nil(t, err) - q := memqueue.NewQueue(logp.L(), nil, memqueue.Settings{Events: 1}) + q := memqueue.NewQueue(logp.L(), nil, memqueue.Settings{Events: 1}, 0) pipeline := makePipeline(Settings{}, q) defer pipeline.Close() diff --git a/libbeat/publisher/pipeline/controller.go b/libbeat/publisher/pipeline/controller.go index bf080677ef44..1c480c01bce2 100644 --- a/libbeat/publisher/pipeline/controller.go +++ b/libbeat/publisher/pipeline/controller.go @@ -62,6 +62,13 @@ type outputController struct { consumer *eventConsumer workers []outputWorker + // The InputQueueSize can be set when the Beat is started, in + // libbeat/cmd/instance/Settings we need to preserve that + // value and pass it into the queue factory. The queue + // factory could be made from elastic-agent output + // configuration reloading which doesn't have access to this + // setting. + inputQueueSize int } type producerRequest struct { @@ -81,6 +88,7 @@ func newOutputController( observer outputObserver, eventWaitGroup *sync.WaitGroup, queueFactory queue.QueueFactory, + inputQueueSize int, ) (*outputController, error) { controller := &outputController{ beat: beat, @@ -90,6 +98,7 @@ func newOutputController( queueFactory: queueFactory, workerChan: make(chan publisher.Batch), consumer: newEventConsumer(monitors.Logger, observer), + inputQueueSize: inputQueueSize, } return controller, nil @@ -258,11 +267,11 @@ func (c *outputController) createQueueIfNeeded(outGrp outputs.Group) { factory = c.queueFactory } - queue, err := factory(logger, c.onACK) + queue, err := factory(logger, c.onACK, c.inputQueueSize) if err != nil { logger.Errorf("queue creation failed, falling back to default memory queue, check your queue configuration") s, _ := memqueue.SettingsForUserConfig(nil) - queue = memqueue.NewQueue(logger, c.onACK, s) + queue = memqueue.NewQueue(logger, c.onACK, s, c.inputQueueSize) } c.queue = queue diff --git a/libbeat/publisher/pipeline/controller_test.go b/libbeat/publisher/pipeline/controller_test.go index 366f4bff1d94..7384e5f71287 100644 --- a/libbeat/publisher/pipeline/controller_test.go +++ b/libbeat/publisher/pipeline/controller_test.go @@ -189,7 +189,7 @@ func TestOutputQueueFactoryTakesPrecedence(t *testing.T) { func TestFailedQueueFactoryRevertsToDefault(t *testing.T) { defaultSettings, _ := memqueue.SettingsForUserConfig(nil) - failedFactory := func(_ *logp.Logger, _ func(int)) (queue.Queue, error) { + failedFactory := func(_ *logp.Logger, _ func(int), _ int) (queue.Queue, error) { return nil, fmt.Errorf("This queue creation intentionally failed") } controller := outputController{ diff --git a/libbeat/publisher/pipeline/pipeline.go b/libbeat/publisher/pipeline/pipeline.go index 209688bb5c2d..cf03163750ee 100644 --- a/libbeat/publisher/pipeline/pipeline.go +++ b/libbeat/publisher/pipeline/pipeline.go @@ -153,13 +153,12 @@ func New( if b := userQueueConfig.Name(); b != "" { queueType = b } - queueFactory, err := queueFactoryForUserConfig( - queueType, userQueueConfig.Config(), settings.InputQueueSize) + queueFactory, err := queueFactoryForUserConfig(queueType, userQueueConfig.Config()) if err != nil { return nil, err } - output, err := newOutputController(beat, monitors, p.observer, p.eventWaitGroup, queueFactory) + output, err := newOutputController(beat, monitors, p.observer, p.eventWaitGroup, queueFactory, settings.InputQueueSize) if err != nil { return nil, err } @@ -399,16 +398,13 @@ func (p *Pipeline) OutputReloader() OutputReloader { // This helper exists to frontload config parsing errors: if there is an // error in the queue config, we want it to show up as fatal during // initialization, even if the queue itself isn't created until later. -func queueFactoryForUserConfig(queueType string, userConfig *conf.C, inQueueSize int) (queue.QueueFactory, error) { +func queueFactoryForUserConfig(queueType string, userConfig *conf.C) (queue.QueueFactory, error) { switch queueType { case memqueue.QueueType: settings, err := memqueue.SettingsForUserConfig(userConfig) if err != nil { return nil, err } - // The memory queue has a special override during pipeline - // initialization for the size of its API channel buffer. - settings.InputQueueSize = inQueueSize return memqueue.FactoryForSettings(settings), nil case diskqueue.QueueType: settings, err := diskqueue.SettingsForUserConfig(userConfig) diff --git a/libbeat/publisher/pipeline/stress/out.go b/libbeat/publisher/pipeline/stress/out.go index 6aa510de1b0e..d1014b8d782b 100644 --- a/libbeat/publisher/pipeline/stress/out.go +++ b/libbeat/publisher/pipeline/stress/out.go @@ -35,11 +35,12 @@ type testOutput struct { } type testOutputConfig struct { - Worker int `config:"worker" validate:"min=1"` - BulkMaxSize int `config:"bulk_max_size"` - Retry int `config:"retry"` - MinWait time.Duration `config:"min_wait"` - MaxWait time.Duration `config:"max_wait"` + Worker int `config:"worker" validate:"min=1"` + BulkMaxSize int `config:"bulk_max_size"` + Retry int `config:"retry"` + MinWait time.Duration `config:"min_wait"` + MaxWait time.Duration `config:"max_wait"` + Queue conf.Namespace `config:"queue"` Fail struct { EveryBatch int } @@ -66,7 +67,7 @@ func makeTestOutput(_ outputs.IndexManager, beat beat.Info, observer outputs.Obs clients[i] = client } - return outputs.Success(config.BulkMaxSize, config.Retry, clients...) + return outputs.Success(config.Queue, config.BulkMaxSize, config.Retry, clients...) } func (*testOutput) Close() error { return nil } diff --git a/libbeat/publisher/queue/diskqueue/queue.go b/libbeat/publisher/queue/diskqueue/queue.go index 2b7548908822..74fff3fea647 100644 --- a/libbeat/publisher/queue/diskqueue/queue.go +++ b/libbeat/publisher/queue/diskqueue/queue.go @@ -109,6 +109,7 @@ func FactoryForSettings(settings Settings) queue.QueueFactory { return func( logger *logp.Logger, ackCallback func(eventCount int), + inputQueueSize int, ) (queue.Queue, error) { return NewQueue(logger, ackCallback, settings) } diff --git a/libbeat/publisher/queue/memqueue/broker.go b/libbeat/publisher/queue/memqueue/broker.go index 0bb3ff9ed8e6..ac5b9dc66159 100644 --- a/libbeat/publisher/queue/memqueue/broker.go +++ b/libbeat/publisher/queue/memqueue/broker.go @@ -84,7 +84,6 @@ type Settings struct { Events int FlushMinEvents int FlushTimeout time.Duration - InputQueueSize int } type queueEntry struct { @@ -123,8 +122,9 @@ func FactoryForSettings(settings Settings) queue.QueueFactory { return func( logger *logp.Logger, ackCallback func(eventCount int), + inputQueueSize int, ) (queue.Queue, error) { - return NewQueue(logger, ackCallback, settings), nil + return NewQueue(logger, ackCallback, settings, inputQueueSize), nil } } @@ -135,6 +135,7 @@ func NewQueue( logger *logp.Logger, ackCallback func(eventCount int), settings Settings, + inputQueueSize int, ) *broker { var ( sz = settings.Events @@ -142,7 +143,7 @@ func NewQueue( flushTimeout = settings.FlushTimeout ) - chanSize := AdjustInputQueueSize(settings.InputQueueSize, sz) + chanSize := AdjustInputQueueSize(inputQueueSize, sz) if minEvents < 1 { minEvents = 1 diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index ef9ee52a9448..28cc38025c38 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -103,7 +103,7 @@ func TestQueueMetricsBuffer(t *testing.T) { } func queueTestWithSettings(t *testing.T, settings Settings, eventsToTest int, testName string) { - testQueue := NewQueue(nil, nil, settings) + testQueue := NewQueue(nil, nil, settings, 0) defer testQueue.Close() // Send events to queue @@ -147,7 +147,7 @@ func makeTestQueue(sz, minEvents int, flushTimeout time.Duration) queuetest.Queu Events: sz, FlushMinEvents: minEvents, FlushTimeout: flushTimeout, - }) + }, 0) } } @@ -258,22 +258,22 @@ func TestEntryIDs(t *testing.T) { } t.Run("acking in forward order with directEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000}) + testQueue := NewQueue(nil, nil, Settings{Events: 1000}, 0) testForward(testQueue) }) t.Run("acking in reverse order with directEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000}) + testQueue := NewQueue(nil, nil, Settings{Events: 1000}, 0) testBackward(testQueue) }) t.Run("acking in forward order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}) + testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}, 0) testForward(testQueue) }) t.Run("acking in reverse order with bufferedEventLoop reports the right event IDs", func(t *testing.T) { - testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}) + testQueue := NewQueue(nil, nil, Settings{Events: 1000, FlushMinEvents: 2, FlushTimeout: time.Microsecond}, 0) testBackward(testQueue) }) } diff --git a/libbeat/publisher/queue/proxy/broker.go b/libbeat/publisher/queue/proxy/broker.go index 20400e3ab75d..832739cc26d9 100644 --- a/libbeat/publisher/queue/proxy/broker.go +++ b/libbeat/publisher/queue/proxy/broker.go @@ -90,6 +90,7 @@ func FactoryForSettings(settings Settings) queue.QueueFactory { return func( logger *logp.Logger, ackCallback func(eventCount int), + inputQueueSize int, ) (queue.Queue, error) { return NewQueue(logger, ackCallback, settings), nil } diff --git a/libbeat/publisher/queue/queue.go b/libbeat/publisher/queue/queue.go index d0e1c0476109..101a32901177 100644 --- a/libbeat/publisher/queue/queue.go +++ b/libbeat/publisher/queue/queue.go @@ -74,7 +74,7 @@ type Queue interface { Metrics() (Metrics, error) } -type QueueFactory func(logger *logp.Logger, ack func(eventCount int)) (Queue, error) +type QueueFactory func(logger *logp.Logger, ack func(eventCount int), inputQueueSize int) (Queue, error) // BufferConfig returns the pipelines buffering settings, // for the pipeline to use. From 42f2f9426043d5ee6fe76ec490e74c9680fdd1b0 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Fri, 20 Oct 2023 10:54:01 +0200 Subject: [PATCH 15/50] [Azure Metrics] Fix CassandraConnectionClosures metric configuration (#34742) * Move CassandraConnectionClosures metrics The name of the metric, "CassandraConnectionClosures" is listed in the wrong section of the metrics configuration. It should be moved to a different section that includes metric names which may or may not be available depending on the environment, with the "ignore_unsupported" setting set to true. * Update changelog * Fix dimensions for CassandraConnectionClosures The `CassandraConnectionClosures` metric only supports the following dimensions: - `ClosureReason` - `Region` --- CHANGELOG.next.asciidoc | 1 + .../module/azure/database_account/manifest.yml | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7c35197567b8..7ca5ca55525e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -139,6 +139,7 @@ is collected by it. - Add remaining dimensions for azure storage account to make them available for tsdb enablement. {pull}36331[36331] - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413] - Add log error when statsd server fails to start {pull}36477[36477] +- Fix CassandraConnectionClosures metric configuration {pull}34742[34742] *Osquerybeat* diff --git a/x-pack/metricbeat/module/azure/database_account/manifest.yml b/x-pack/metricbeat/module/azure/database_account/manifest.yml index 3436008db7ee..1ef2ecb9b2cc 100644 --- a/x-pack/metricbeat/module/azure/database_account/manifest.yml +++ b/x-pack/metricbeat/module/azure/database_account/manifest.yml @@ -9,7 +9,7 @@ input: resource_type: "Microsoft.DocumentDb/databaseAccounts" metrics: - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency", - "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures", "UpdateDiagnosticsSettings"] + "RegionFailover", "DeleteAccount", "CreateAccount", "UpdateDiagnosticsSettings"] namespace: "Microsoft.DocumentDb/databaseAccounts" - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] @@ -35,6 +35,13 @@ input: dimensions: - name: "DatabaseName" value: "*" + - name: ["CassandraConnectionClosures"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true + timegrain: "PT1M" + dimensions: + - name: "ClosureReason" + value: "*" - name: [ "GremlinDatabaseDelete", "GremlinDatabaseThroughputUpdate", "GremlinDatabaseUpdate", "GremlinGraphDelete","GremlinGraphThroughputUpdate", "GremlinGraphUpdate", "MongoCollectionDelete", "MongoCollectionThroughputUpdate", "MongoCollectionUpdate", "MongoDBDatabaseUpdate", "MongoDatabaseDelete", "MongoDatabaseThroughputUpdate", "CassandraKeyspaceDelete", "CassandraKeyspaceThroughputUpdate", "CassandraKeyspaceUpdate","CassandraTableDelete", "CassandraTableThroughputUpdate", "CassandraTableUpdate", @@ -48,7 +55,7 @@ input: - resource_id: "" metrics: - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency", - "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures", "UpdateDiagnosticsSettings"] + "RegionFailover", "DeleteAccount", "CreateAccount", "UpdateDiagnosticsSettings"] namespace: "Microsoft.DocumentDb/databaseAccounts" - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] @@ -74,6 +81,13 @@ input: dimensions: - name: "DatabaseName" value: "*" + - name: ["CassandraConnectionClosures"] + namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true + timegrain: "PT1M" + dimensions: + - name: "ClosureReason" + value: "*" - name: [ "GremlinDatabaseDelete", "GremlinDatabaseThroughputUpdate", "GremlinDatabaseUpdate", "GremlinGraphDelete","GremlinGraphThroughputUpdate", "GremlinGraphUpdate", "MongoCollectionDelete", "MongoCollectionThroughputUpdate", "MongoCollectionUpdate", "MongoDBDatabaseUpdate", "MongoDatabaseDelete", "MongoDatabaseThroughputUpdate", "CassandraKeyspaceDelete", "CassandraKeyspaceThroughputUpdate", "CassandraKeyspaceUpdate","CassandraTableDelete", "CassandraTableThroughputUpdate", "CassandraTableUpdate", From 11f4dee7f5077b07ed355955478a55a89233805e Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:22:47 +1030 Subject: [PATCH 16/50] x-pack/filebeat/docs/inputs/cel: fix basic auth example (#36903) The previous example refers to the httpjson mechanism. --- x-pack/filebeat/docs/inputs/input-cel.asciidoc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index 35fbc4654f22..aef0c4862ab1 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -247,11 +247,10 @@ Example configurations with authentication: ---- filebeat.inputs: - type: cel + auth.basic: + user: user@domain.tld + password: P@$$W0₹D resource.url: http://localhost - request.transforms: - - set: - target: header.Authorization - value: 'Basic aGVsbG86d29ybGQ=' ---- ["source","yaml",subs="attributes"] From 09823f372b68c22753310328dc8586a3535b6b9a Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Mon, 23 Oct 2023 09:22:03 +0200 Subject: [PATCH 17/50] Update go grpc version to 1.58.3 (#36904) --- NOTICE.txt | 734 +++++++++++++++++++++++++++++++++++++++++++++++++---- go.mod | 45 ++-- go.sum | 97 ++++--- 3 files changed, 765 insertions(+), 111 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 6ae9f58daba1..019668a2a8fd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -11,11 +11,11 @@ Third party libraries used by the Elastic Beats project: -------------------------------------------------------------------------------- Dependency : cloud.google.com/go -Version: v0.107.0 +Version: v0.110.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go@v0.107.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go@v0.110.4/LICENSE: Apache License @@ -223,11 +223,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go@v0.107.0/LICEN -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/bigquery -Version: v1.44.0 +Version: v1.52.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/bigquery@v1.44.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/bigquery@v1.52.0/LICENSE: Apache License @@ -435,11 +435,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/bigquery@v1.44 -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/compute -Version: v1.15.1 +Version: v1.21.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/compute@v1.15.1/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/compute@v1.21.0/LICENSE: Apache License @@ -647,11 +647,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/compute@v1.15. -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/monitoring -Version: v1.9.0 +Version: v1.15.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/monitoring@v1.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/monitoring@v1.15.1/LICENSE: Apache License @@ -859,11 +859,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/monitoring@v1. -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/pubsub -Version: v1.27.1 +Version: v1.32.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/pubsub@v1.27.1/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/pubsub@v1.32.0/LICENSE: Apache License @@ -1071,11 +1071,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/pubsub@v1.27.1 -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/redis -Version: v1.10.0 +Version: v1.13.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/redis@v1.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/redis@v1.13.1/LICENSE: Apache License @@ -1283,11 +1283,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/redis@v1.10.0/ -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/storage -Version: v1.27.0 +Version: v1.30.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/storage@v1.27.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/storage@v1.30.1/LICENSE: Apache License @@ -17325,11 +17325,11 @@ Contents of probable licence file $GOMODCACHE/github.com/golang/mock@v1.6.0/LICE -------------------------------------------------------------------------------- Dependency : github.com/golang/protobuf -Version: v1.5.2 +Version: v1.5.3 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.3/LICENSE: Copyright 2010 The Go Authors. All rights reserved. @@ -18154,11 +18154,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/googleapis/gax-go/v2 -Version: v2.7.0 +Version: v2.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/googleapis/gax-go/v2@v2.7.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/googleapis/gax-go/v2@v2.11.0/LICENSE: Copyright 2016, Google Inc. All rights reserved. @@ -24781,11 +24781,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/mod -Version: v0.9.0 +Version: v0.10.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.10.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24818,11 +24818,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.10.0 +Version: v0.12.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.12.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24855,11 +24855,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/oauth2 -Version: v0.7.0 +Version: v0.10.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/oauth2@v0.7.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/oauth2@v0.10.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24892,11 +24892,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sync -Version: v0.1.0 +Version: v0.3.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.3.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25040,11 +25040,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/tools -Version: v0.6.0 +Version: v0.9.1 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.9.1/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25077,11 +25077,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : google.golang.org/api -Version: v0.103.0 +Version: v0.126.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/api@v0.103.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/api@v0.126.0/LICENSE: Copyright (c) 2011 Google Inc. All rights reserved. @@ -25113,12 +25113,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : google.golang.org/genproto -Version: v0.0.0-20230110181048-76db0878b65f +Dependency : google.golang.org/genproto/googleapis/api +Version: v0.0.0-20230711160842-782d3b101e98 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0-20230110181048-76db0878b65f/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/genproto/googleapis/api@v0.0.0-20230711160842-782d3b101e98/LICENSE: Apache License @@ -25326,11 +25326,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0- -------------------------------------------------------------------------------- Dependency : google.golang.org/grpc -Version: v1.53.0 +Version: v1.58.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.53.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.58.3/LICENSE: Apache License @@ -25538,11 +25538,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/grpc@v1.53.0/LIC -------------------------------------------------------------------------------- Dependency : google.golang.org/protobuf -Version: v1.29.1 +Version: v1.31.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.29.1/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.31.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -27659,11 +27659,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/compute/metada -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/datacatalog -Version: v1.8.0 +Version: v1.14.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/datacatalog@v1.8.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/datacatalog@v1.14.1/LICENSE: Apache License @@ -27871,11 +27871,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/datacatalog@v1 -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/iam -Version: v0.8.0 +Version: v1.1.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/iam@v0.8.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/iam@v1.1.1/LICENSE: Apache License @@ -28083,11 +28083,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/iam@v0.8.0/LIC -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/kms -Version: v1.6.0 +Version: v1.12.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/kms@v1.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/kms@v1.12.1/LICENSE: Apache License @@ -28295,11 +28295,11 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/kms@v1.6.0/LIC -------------------------------------------------------------------------------- Dependency : cloud.google.com/go/longrunning -Version: v0.3.0 +Version: v0.5.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/longrunning@v0.3.0/LICENSE: +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/longrunning@v0.5.1/LICENSE: Apache License @@ -38185,11 +38185,11 @@ Contents of probable licence file $GOMODCACHE/github.com/google/martian@v2.1.0+i -------------------------------------------------------------------------------- Dependency : github.com/google/martian/v3 -Version: v3.2.1 +Version: v3.3.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/google/martian/v3@v3.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/google/martian/v3@v3.3.2/LICENSE: Apache License @@ -38607,6 +38607,218 @@ Contents of probable licence file $GOMODCACHE/github.com/google/pprof@v0.0.0-202 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/google/s2a-go +Version: v0.1.4 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/google/s2a-go@v0.1.4/LICENSE.md: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/google/shlex Version: v0.0.0-20191202100458-e7afc7fbc510 @@ -38821,11 +39033,11 @@ Contents of probable licence file $GOMODCACHE/github.com/google/shlex@v0.0.0-201 -------------------------------------------------------------------------------- Dependency : github.com/googleapis/enterprise-certificate-proxy -Version: v0.2.0 +Version: v0.2.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/googleapis/enterprise-certificate-proxy@v0.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/googleapis/enterprise-certificate-proxy@v0.2.3/LICENSE: Apache License @@ -51180,6 +51392,430 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/appengine@v1.6.7 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : google.golang.org/genproto +Version: v0.0.0-20230711160842-782d3b101e98 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/google.golang.org/genproto@v0.0.0-20230711160842-782d3b101e98/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : google.golang.org/genproto/googleapis/rpc +Version: v0.0.0-20230711160842-782d3b101e98 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/google.golang.org/genproto/googleapis/rpc@v0.0.0-20230711160842-782d3b101e98/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : gopkg.in/check.v1 Version: v1.0.0-20201130134442-10cb98267c6c diff --git a/go.mod b/go.mod index e4955abcc912..fc3e8eab6869 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/elastic/beats/v7 go 1.20 require ( - cloud.google.com/go/bigquery v1.44.0 - cloud.google.com/go/monitoring v1.9.0 - cloud.google.com/go/pubsub v1.27.1 + cloud.google.com/go/bigquery v1.52.0 + cloud.google.com/go/monitoring v1.15.1 + cloud.google.com/go/pubsub v1.32.0 code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee // indirect code.cloudfoundry.org/go-loggregator v7.4.0+incompatible code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a // indirect @@ -95,7 +95,7 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 github.com/gomodule/redigo v1.8.3 github.com/google/flatbuffers v23.3.3+incompatible @@ -154,18 +154,18 @@ require ( go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/mod v0.9.0 - golang.org/x/net v0.10.0 - golang.org/x/oauth2 v0.7.0 - golang.org/x/sync v0.1.0 + golang.org/x/mod v0.10.0 + golang.org/x/net v0.12.0 + golang.org/x/oauth2 v0.10.0 + golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 golang.org/x/text v0.12.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.6.0 - google.golang.org/api v0.103.0 - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f - google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.29.1 + golang.org/x/tools v0.9.1 + google.golang.org/api v0.126.0 + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.3 + google.golang.org/protobuf v1.31.0 gopkg.in/inf.v0 v0.9.1 gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect @@ -182,9 +182,9 @@ require ( ) require ( - cloud.google.com/go v0.107.0 - cloud.google.com/go/compute v1.15.1 - cloud.google.com/go/redis v1.10.0 + cloud.google.com/go v0.110.4 + cloud.google.com/go/compute v1.21.0 + cloud.google.com/go/redis v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.0.0 @@ -210,7 +210,7 @@ require ( github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 github.com/google/cel-go v0.15.3 - github.com/googleapis/gax-go/v2 v2.7.0 + github.com/googleapis/gax-go/v2 v2.11.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/lestrrat-go/jwx/v2 v2.0.11 @@ -223,14 +223,15 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.4.4 go.elastic.co/apm/v2 v2.4.4 go.mongodb.org/mongo-driver v1.5.1 + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( aqwari.net/xml v0.0.0-20210331023308-d9421b293817 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.8.0 // indirect - cloud.google.com/go/longrunning v0.3.0 // indirect + cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect github.com/Azure/azure-amqp-common-go/v3 v3.2.1 // indirect github.com/Azure/azure-pipeline-go v0.2.1 // indirect @@ -288,8 +289,9 @@ require ( github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/cronexpr v1.1.0 // indirect @@ -365,6 +367,7 @@ require ( golang.org/x/term v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.30.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect @@ -377,7 +380,7 @@ require ( ) require ( - cloud.google.com/go/storage v1.27.0 + cloud.google.com/go/storage v1.30.1 github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/go.sum b/go.sum index a1aecdb59dfd..47cb095833b3 100644 --- a/go.sum +++ b/go.sum @@ -24,47 +24,47 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= -cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.44.0 h1:Wi4dITi+cf9VYp4VH2T9O41w0kCW0uQTELq2Z6tukN0= -cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.52.0 h1:JKLNdxI0N+TIUWD6t9KN646X27N5dQWq9dZbbTWZ8hc= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.3.0/go.mod h1:z5EyKrPE8OQmeg4h5MNdKvuSnI9CCT49Ki3f23aBzio= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.8.0 h1:6kZ4RIOW/uT7QWC5SfPfq/G8sYzr/v+UOmOAxy4Z1TE= +cloud.google.com/go/datacatalog v1.14.1 h1:cFPBt8V5V2T3mu/96tc4nhcMB+5cYcpwjBfn79bZDI8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/kms v1.6.0 h1:OWRZzrPmOZUzurjI2FBGtgY2mB1WaJkqhw6oIwSj0Yg= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= -cloud.google.com/go/monitoring v1.9.0 h1:O2A5HsrhvRMzD3OMUimPXF46vOzwc9vh6oGCGf9i/ws= -cloud.google.com/go/monitoring v1.9.0/go.mod h1:/FsTS0gkEFUc4cgB16s6jYDnyjzRBkRJNRzBn5Zx+wA= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.27.1 h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs= -cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/redis v1.10.0 h1:/zTwwBKIAD2DEWTrXZp8WD9yD/gntReF/HkPssVYd0U= -cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/redis v1.13.1 h1:YrjQnCC7ydk+k30op7DSjSHw1yAYhqYXFcOq1bSXRYA= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= -cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee h1:iAAPf9s7/+BIiGf+RjgcXLm3NoZaLIJsBXJuUa63Lx8= code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee/go.mod h1:Jzi+ccHgo/V/PLQUaQ6hnZcC1c4BS790gx21LRRui4g= code.cloudfoundry.org/go-loggregator v7.4.0+incompatible h1:KqZYloMQWM5Zg/BQKunOIA4OODh7djZbk48qqbowNFI= @@ -438,6 +438,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -716,6 +717,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -1004,8 +1006,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -1053,8 +1056,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -1071,6 +1074,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230426061923-93006964c1fc h1:AGDHt781oIcL4EFk7cPnvBUYTwU8BEU6GDTO3ZMn1sE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1079,12 +1084,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= @@ -2016,6 +2021,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -2079,8 +2085,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2162,8 +2168,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2178,8 +2185,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2195,8 +2202,9 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2353,6 +2361,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -2460,8 +2469,9 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2506,8 +2516,8 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2571,8 +2581,12 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2604,9 +2618,10 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2622,8 +2637,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 185274767e766fc12128cea41f8bc3e981e3238d Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:53:15 +1030 Subject: [PATCH 18/50] x-pack/winlogbeat/module/routing: make pipeline routing robust to channel letter case (#36899) Apparently some events from Windows servers and workstations in Security channel have a lowercase channel name. This has not been observed in other channels, but defensively apply the same care there. --- CHANGELOG.next.asciidoc | 1 + x-pack/winlogbeat/module/routing/ingest/routing.yml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7ca5ca55525e..56e207a7afe4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -274,6 +274,7 @@ is collected by it. *Winlogbeat* +- Make ingest pipeline routing robust to letter case of channel names for forwarded events. {issue}36670[36670] {pull}36899[36899] *Functionbeat* diff --git a/x-pack/winlogbeat/module/routing/ingest/routing.yml b/x-pack/winlogbeat/module/routing/ingest/routing.yml index 1ef2769c0b94..9c00e19e1603 100644 --- a/x-pack/winlogbeat/module/routing/ingest/routing.yml +++ b/x-pack/winlogbeat/module/routing/ingest/routing.yml @@ -6,16 +6,16 @@ processors: value: '{{_ingest.timestamp}}' - pipeline: name: '{< IngestPipeline "security" >}' - if: ctx?.winlog?.channel == 'Security' && ['Microsoft-Windows-Eventlog', 'Microsoft-Windows-Security-Auditing'].contains(ctx?.winlog?.provider_name) + if: ctx.winlog?.channel instanceof String && ctx.winlog.channel.toLowerCase() == 'security' && ['Microsoft-Windows-Eventlog', 'Microsoft-Windows-Security-Auditing'].contains(ctx.winlog?.provider_name) - pipeline: name: '{< IngestPipeline "sysmon" >}' - if: ctx?.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' + if: ctx.winlog?.channel instanceof String && ctx.winlog.channel.toLowerCase() == 'microsoft-windows-sysmon/operational' - pipeline: name: '{< IngestPipeline "powershell" >}' - if: ctx?.winlog?.channel == 'Windows PowerShell' + if: ctx.winlog?.channel instanceof String && ctx.winlog.channel.toLowerCase() == 'windows powershell' - pipeline: name: '{< IngestPipeline "powershell_operational" >}' - if: ctx?.winlog?.channel == 'Microsoft-Windows-PowerShell/Operational' + if: ctx.winlog?.channel instanceof String && ctx.winlog.channel.toLowerCase() == 'microsoft-windows-powershell/operational' - set: field: host.os.type value: windows From 07ad78e4e218125a46540c1b2b7cfc942d987e38 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 23 Oct 2023 09:45:53 -0400 Subject: [PATCH 19/50] [Cleanup] Remove unused File type --- filebeat/input/file/file.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/filebeat/input/file/file.go b/filebeat/input/file/file.go index 961f1476a77f..87e8efe73bcf 100644 --- a/filebeat/input/file/file.go +++ b/filebeat/input/file/file.go @@ -23,13 +23,6 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type File struct { - File *os.File - FileInfo os.FileInfo - Path string - State *State -} - // IsSameFile checks if the given File path corresponds with the FileInfo given // It is used to check if the file has been renamed. func IsSameFile(path string, info os.FileInfo) bool { From 2b7dbeaa53d5b905fc39e3f7f68073b9a3062574 Mon Sep 17 00:00:00 2001 From: Brian Dols Date: Mon, 23 Oct 2023 09:11:39 -0500 Subject: [PATCH 20/50] Raise logging level for autodiscover configuration errors Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscover events/environments. --- CHANGELOG.next.asciidoc | 1 + libbeat/autodiscover/template/config.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 56e207a7afe4..890675b2385f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -169,6 +169,7 @@ is collected by it. Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor - allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788] - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] +- Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments *Auditbeat* diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index c050ff8acd86..3ba0db210de9 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -154,7 +154,7 @@ func ApplyConfigTemplate(event bus.Event, configs []*conf.C, options ...ucfg.Opt var unpacked map[string]interface{} err = c.Unpack(&unpacked, opts...) if err != nil { - logp.Debug("autodiscover", "Configuration template cannot be resolved: %v", err) + logp.Warn("autodiscover: Configuration template cannot be resolved: %v", err) continue } // Repack again: From c143fa6f3c854a60d4714d8531cdfc069452e018 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:13:07 -0700 Subject: [PATCH 21/50] Serverless user messages (#36944) * separate out validation messages in index setup * add extra line --- libbeat/idxmgmt/index_support.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libbeat/idxmgmt/index_support.go b/libbeat/idxmgmt/index_support.go index 94fd3b07db08..4526d916c359 100644 --- a/libbeat/idxmgmt/index_support.go +++ b/libbeat/idxmgmt/index_support.go @@ -227,7 +227,12 @@ func (m *indexManager) VerifySetup(loadTemplate, loadLifecycle LoadMode) (bool, if !ilmComponent.load { warn += "lifecycle policy loading not enabled.\n" } else if !ilmComponent.overwrite { - warn += "Overwriting lifecycle policy is disabled. Set `setup.ilm.overwrite: true` or `setup.dsl.overwrite: true` to overwrite.\n" + if m.clientHandler.Mode() == lifecycle.DSL { + warn += "Overwriting lifecycle policy is disabled. Set `setup.dsl.overwrite: true` to overwrite.\n" + } else { + warn += "Overwriting lifecycle policy is disabled. Set `setup.ilm.overwrite: true` to overwrite.\n" + } + } if !templateComponent.load { warn += "Template loading not enabled.\n" From 58b2de1e58aee986279b7c7b88d88cfc76394640 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:10:45 +1030 Subject: [PATCH 22/50] x-pack/filebeat/input/cel: add support for digest authentication (#36932) --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 31 ++++ go.mod | 1 + go.sum | 2 + .../filebeat/docs/inputs/input-cel.asciidoc | 36 ++++- x-pack/filebeat/input/cel/config_auth.go | 46 +++++- x-pack/filebeat/input/cel/config_test.go | 72 +++++++-- x-pack/filebeat/input/cel/input.go | 14 ++ x-pack/filebeat/input/cel/input_test.go | 148 ++++++++++++++++++ 9 files changed, 331 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 890675b2385f..7fbe4c0e480d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -243,6 +243,7 @@ is collected by it. - Add cache processor. {pull}36786[36786] - Avoid unwanted publication of Azure entity records. {pull}36753[36753] - Avoid unwanted publication of Okta entity records. {pull}36770[36770] +- Add support for Digest Authentication to CEL input. {issue}35514[35514] {pull}36932[36932] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 019668a2a8fd..7061208f1ef6 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -20004,6 +20004,37 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/icholy/digest +Version: v0.1.22 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/icholy/digest@v0.1.22/LICENSE: + +MIT License + +Copyright (c) 2020 Ilia Choly + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/dhcp Version: v0.0.0-20200227161230-57ec251c7eb3 diff --git a/go.mod b/go.mod index fc3e8eab6869..227b7659e633 100644 --- a/go.mod +++ b/go.mod @@ -213,6 +213,7 @@ require ( github.com/googleapis/gax-go/v2 v2.11.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 + github.com/icholy/digest v0.1.22 github.com/lestrrat-go/jwx/v2 v2.0.11 github.com/otiai10/copy v1.12.0 github.com/pierrec/lz4/v4 v4.1.16 diff --git a/go.sum b/go.sum index 47cb095833b3..6ce895072b50 100644 --- a/go.sum +++ b/go.sum @@ -1201,6 +1201,8 @@ github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbc github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/icholy/digest v0.1.22 h1:dRIwCjtAcXch57ei+F0HSb5hmprL873+q7PoVojdMzM= +github.com/icholy/digest v0.1.22/go.mod h1:uLAeDdWKIWNFMH0wqbwchbTQOmJWhzSnL7zmqSPqEEc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= diff --git a/x-pack/filebeat/docs/inputs/input-cel.asciidoc b/x-pack/filebeat/docs/inputs/input-cel.asciidoc index aef0c4862ab1..786f98aa842f 100644 --- a/x-pack/filebeat/docs/inputs/input-cel.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cel.asciidoc @@ -22,6 +22,7 @@ This input supports: * Auth ** Basic +** Digest ** OAuth2 * Retrieval at a configurable interval * Pagination @@ -239,7 +240,7 @@ As noted above the `cel` input provides functions, macros, and global variables In addition to the extensions provided in the packages listed above, a global variable `useragent` is also provided which gives the user CEL program access to the {beatname_lc} user-agent string. -Additionally, it supports authentication via Basic auth, HTTP Headers or oauth2. +Additionally, it supports authentication via Basic Authentication, Digest Authentication or OAuth2. Example configurations with authentication: @@ -253,6 +254,16 @@ filebeat.inputs: resource.url: http://localhost ---- +["source","yaml",subs="attributes"] +---- +filebeat.inputs: +- type: cel + auth.digest: + user: user@domain.tld + password: P@$$W0₹D + resource.url: http://localhost +---- + ["source","yaml",subs="attributes"] ---- filebeat.inputs: @@ -398,6 +409,29 @@ The user to authenticate with. The password to use. +[float] +==== `auth.digest.enabled` + +When set to `false`, disables the digest auth configuration. Default: `true`. + +NOTE: digest auth settings are disabled if either `enabled` is set to `false` or +the `auth.digest` section is missing. + +[float] +==== `auth.digest.user` + +The user to authenticate with. + +[float] +==== `auth.digest.password` + +The password to use. + +[float] +==== `auth.digest.no_reuse` + +When set to `true`, Digest Authentication challenges are not reused. + [float] ==== `auth.oauth2.enabled` diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index 5c80507fdadd..e550a9635d51 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -24,12 +24,23 @@ import ( ) type authConfig struct { - Basic *basicAuthConfig `config:"basic"` - OAuth2 *oAuth2Config `config:"oauth2"` + Basic *basicAuthConfig `config:"basic"` + Digest *digestAuthConfig `config:"digest"` + OAuth2 *oAuth2Config `config:"oauth2"` } func (c authConfig) Validate() error { - if c.Basic.isEnabled() && c.OAuth2.isEnabled() { + var n int + if c.Basic.isEnabled() { + n++ + } + if c.Digest.isEnabled() { + n++ + } + if c.OAuth2.isEnabled() { + n++ + } + if n > 1 { return errors.New("only one kind of auth can be enabled") } return nil @@ -41,7 +52,7 @@ type basicAuthConfig struct { Password string `config:"password"` } -// IsEnabled returns true if the `enable` field is set to true in the yaml. +// isEnabled returns true if the `enable` field is set to true in the yaml. func (b *basicAuthConfig) isEnabled() bool { return b != nil && (b.Enabled == nil || *b.Enabled) } @@ -59,6 +70,31 @@ func (b *basicAuthConfig) Validate() error { return nil } +type digestAuthConfig struct { + Enabled *bool `config:"enabled"` + User string `config:"user"` + Password string `config:"password"` + NoReuse *bool `config:"no_reuse"` +} + +// isEnabled returns true if the `enable` field is set to true in the yaml. +func (d *digestAuthConfig) isEnabled() bool { + return d != nil && (d.Enabled == nil || *d.Enabled) +} + +// Validate checks if oauth2 config is valid. +func (d *digestAuthConfig) Validate() error { + if !d.isEnabled() { + return nil + } + + if d.User == "" || d.Password == "" { + return errors.New("both user and password must be set") + } + + return nil +} + // An oAuth2Provider represents a supported oauth provider. type oAuth2Provider string @@ -107,7 +143,7 @@ type oAuth2Config struct { OktaJWKJSON common.JSONBlob `config:"okta.jwk_json"` } -// IsEnabled returns true if the `enable` field is set to true in the yaml. +// isEnabled returns true if the `enable` field is set to true in the yaml. func (o *oAuth2Config) isEnabled() bool { return o != nil && (o.Enabled == nil || *o.Enabled) } diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index e19c08399cbf..0cd404705e2d 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -39,26 +39,45 @@ func TestGetProviderIsCanonical(t *testing.T) { } func TestIsEnabled(t *testing.T) { - oauth2 := oAuth2Config{} - if !oauth2.isEnabled() { - t.Errorf("OAuth2 not enabled by default") + type enabler interface { + isEnabled() bool + take(*bool) } + for _, test := range []struct { + name string + auth enabler + }{ + {name: "basic", auth: &basicAuthConfig{}}, + {name: "digest", auth: &digestAuthConfig{}}, + {name: "OAuth2", auth: &oAuth2Config{}}, + } { + t.Run(test.name, func(t *testing.T) { + if !test.auth.isEnabled() { + t.Errorf("auth not enabled by default") + } - var enabled bool - for i := 0; i < 4; i++ { - oauth2.Enabled = &enabled - if got := oauth2.isEnabled(); got != enabled { - t.Errorf("unexpected OAuth2 enabled state on iteration %d: got:%t want:%t", i, got, enabled) - } - enabled = !enabled - } + var enabled bool + for i := 0; i < 4; i++ { + test.auth.take(&enabled) + if got := test.auth.isEnabled(); got != enabled { + t.Errorf("unexpected auth enabled state on iteration %d: got:%t want:%t", i, got, enabled) + } + enabled = !enabled + } - oauth2.Enabled = nil - if !oauth2.isEnabled() { - t.Errorf("OAuth2 not enabled if nilled") + test.auth.take(nil) + if !test.auth.isEnabled() { + t.Errorf("auth not enabled if nilled") + } + }) } } +// take methods are for testing only. +func (b *basicAuthConfig) take(on *bool) { b.Enabled = on } +func (d *digestAuthConfig) take(on *bool) { d.Enabled = on } +func (o *oAuth2Config) take(on *bool) { o.Enabled = on } + func TestOAuth2GetTokenURL(t *testing.T) { const host = "http://localhost" for _, test := range []struct { @@ -143,6 +162,31 @@ var oAuth2ValidationTests = []struct { }, }, }, + { + name: "can't_set_oauth2_and_digest_auth_together", + wantErr: errors.New("only one kind of auth can be enabled accessing 'auth'"), + input: map[string]interface{}{ + "auth.digest.user": "user", + "auth.digest.password": "pass", + "auth.oauth2": map[string]interface{}{ + "token_url": "localhost", + "client": map[string]interface{}{ + "id": "a_client_id", + "secret": "a_client_secret", + }, + }, + }, + }, + { + name: "can't_set_basic_and_digest_auth_together", + wantErr: errors.New("only one kind of auth can be enabled accessing 'auth'"), + input: map[string]interface{}{ + "auth.basic.user": "user", + "auth.basic.password": "pass", + "auth.digest.user": "user", + "auth.digest.password": "pass", + }, + }, { name: "can_set_oauth2_and_basic_auth_together_if_oauth2_is_disabled", input: map[string]interface{}{ diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index ea4958b9c30e..499ae97f7fac 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -24,6 +24,7 @@ import ( "time" retryablehttp "github.com/hashicorp/go-retryablehttp" + "github.com/icholy/digest" "github.com/rcrowley/go-metrics" "go.elastic.co/ecszap" "go.uber.org/zap" @@ -688,6 +689,19 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, return nil, err } + if cfg.Auth.Digest.isEnabled() { + var noReuse bool + if cfg.Auth.Digest.NoReuse != nil { + noReuse = *cfg.Auth.Digest.NoReuse + } + c.Transport = &digest.Transport{ + Transport: c.Transport, + Username: cfg.Auth.Digest.User, + Password: cfg.Auth.Digest.Password, + NoReuse: noReuse, + } + } + if cfg.Resource.Tracer != nil { w := zapcore.AddSync(cfg.Resource.Tracer) go func() { diff --git a/x-pack/filebeat/input/cel/input_test.go b/x-pack/filebeat/input/cel/input_test.go index bc0cf1714713..1a0a7b442114 100644 --- a/x-pack/filebeat/input/cel/input_test.go +++ b/x-pack/filebeat/input/cel/input_test.go @@ -7,6 +7,7 @@ package cel import ( "context" + "flag" "fmt" "io" "math/rand" @@ -22,6 +23,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/icholy/digest" v2 "github.com/elastic/beats/v7/filebeat/input/v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" @@ -31,8 +33,11 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +var runRemote = flag.Bool("run_remote", false, "run tests using remote endpoints") + var inputTests = []struct { name string + remote bool server func(*testing.T, http.HandlerFunc, map[string]interface{}) handler http.HandlerFunc config map[string]interface{} @@ -1055,6 +1060,100 @@ var inputTests = []struct { }, // Authenticated access tests. + { + name: "digest_accept", + server: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + s := httptest.NewServer(h) + config["resource.url"] = s.URL + t.Cleanup(s.Close) + }, + config: map[string]interface{}{ + "interval": 1, + "auth.digest.user": "test_client", + "auth.digest.password": "secret_password", + "program": ` + bytes(get(state.url).Body).as(body, { + "events": [body.decode_json()] + }) + `, + }, + handler: digestAuthHandler( + "test_client", + "secret_password", + "test", + "random_string", + defaultHandler(http.MethodGet, ""), + ), + want: []map[string]interface{}{ + { + "hello": []interface{}{ + map[string]interface{}{ + "world": "moon", + }, + map[string]interface{}{ + "space": []interface{}{ + map[string]interface{}{ + "cake": "pumpkin", + }, + }, + }, + }, + }, + }, + }, + { + name: "digest_reject", + server: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { + s := httptest.NewServer(h) + config["resource.url"] = s.URL + t.Cleanup(s.Close) + }, + config: map[string]interface{}{ + "interval": 1, + "auth.digest.user": "test_client", + "auth.digest.password": "wrong_secret_password", + "program": ` + bytes(get(state.url).Body).as(body, { + "events": [body.decode_json()] + }) + `, + }, + handler: digestAuthHandler( + "test_client", + "secret_password", + "test", + "random_string", + defaultHandler(http.MethodGet, ""), + ), + want: []map[string]interface{}{ + { + "error": "not authorized", + }, + }, + }, + { + // Test case modelled on `curl --digest -u test_user:secret_password https://httpbin.org/digest-auth/auth/test_user/secret_password/md5`. + name: "digest_remote", + remote: true, + server: func(_ *testing.T, _ http.HandlerFunc, _ map[string]interface{}) {}, + config: map[string]interface{}{ + "resource.url": "https://httpbin.org/digest-auth/auth/test_user/secret_password/md5", + "interval": 1, + "auth.digest.user": "test_user", + "auth.digest.password": "secret_password", + "program": ` + bytes(get(state.url).Body).as(body, { + "events": [body.decode_json()] + }) + `, + }, + want: []map[string]interface{}{ + { + "authenticated": true, + "user": "test_user", + }, + }, + }, { name: "OAuth2", server: func(t *testing.T, h http.HandlerFunc, config map[string]interface{}) { @@ -1248,6 +1347,9 @@ func TestInput(t *testing.T) { if reason, skip := skipOnWindows[test.name]; runtime.GOOS == "windows" && skip { t.Skip(reason) } + if test.remote && !*runRemote { + t.Skip("skipping remote endpoint test") + } if test.server != nil { test.server(t, test.handler, test.config) @@ -1482,6 +1584,52 @@ func retryHandler() http.HandlerFunc { } } +//nolint:errcheck // No point checking errors in test server. +func digestAuthHandler(user, pass, realm, nonce string, handle http.HandlerFunc) http.HandlerFunc { + chal := &digest.Challenge{ + Realm: realm, + Nonce: nonce, + Algorithm: "MD5", + QOP: []string{"auth"}, + } + return func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth == "" { + w.Header().Add("WWW-Authenticate", chal.String()) + w.WriteHeader(http.StatusUnauthorized) + return + } + + reqCred, err := digest.ParseCredentials(auth) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + srvCred, err := digest.Digest(chal, digest.Options{ + Method: r.Method, + URI: r.URL.RequestURI(), + Cnonce: reqCred.Cnonce, + Count: reqCred.Nc, + Username: user, + Password: pass, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if reqCred.Response != srvCred.Response { + w.Header().Set("content-type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte(`{"error":"not authorized"}`)) + return + } + + handle(w, r) + } +} + //nolint:errcheck // No point checking errors in test server. func oauth2Handler(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/token" { From 546a413de54a13e19821093f9b7a84f1ca36b25d Mon Sep 17 00:00:00 2001 From: Denis Date: Tue, 24 Oct 2023 09:23:00 +0200 Subject: [PATCH 23/50] Refactor `Event` type and add more test coverage (#36906) * Added tests with full coverage * Standardized the behavior with `mapstr.M`. --- libbeat/beat/event.go | 234 ++++--- libbeat/beat/event_test.go | 810 ++++++++++++++--------- libbeat/processors/add_id/add_id_test.go | 4 +- 3 files changed, 675 insertions(+), 373 deletions(-) diff --git a/libbeat/beat/event.go b/libbeat/beat/event.go index 54fc3e27cc39..dffb827603aa 100644 --- a/libbeat/beat/event.go +++ b/libbeat/beat/event.go @@ -19,6 +19,7 @@ package beat import ( "errors" + "fmt" "strings" "time" @@ -26,12 +27,22 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +type updateMode bool + +var ( + updateModeOverwrite updateMode = true + updateModeNoOverwrite updateMode = false +) + // FlagField fields used to keep information or errors when events are parsed. const FlagField = "log.flags" const ( - timestampFieldKey = "@timestamp" - metadataFieldKey = "@metadata" + TimestampFieldKey = "@timestamp" + MetadataFieldKey = "@metadata" + ErrorFieldKey = "error" + metadataKeyPrefix = MetadataFieldKey + "." + metadataKeyOffset = len(metadataKeyPrefix) ) // Event is the common event format shared by all beats. @@ -47,28 +58,44 @@ type Event struct { } var ( - errNoTimestamp = errors.New("value is no timestamp") - errNoMapStr = errors.New("value is no map[string]interface{} type") + ErrValueNotTimestamp = errors.New("value is not a timestamp") + ErrValueNotMapStr = errors.New("value is not `mapstr.M` or `map[string]interface{}` type") + ErrAlterMetadataKey = fmt.Errorf("deleting/replacing %q key is not supported", MetadataFieldKey) + ErrMetadataAccess = fmt.Errorf("accessing %q key directly is not supported, try nested keys", MetadataFieldKey) + ErrDeleteTimestamp = fmt.Errorf("deleting %q key is not supported", TimestampFieldKey) ) // SetID overwrites the "id" field in the events metadata. // If Meta is nil, a new Meta dictionary is created. func (e *Event) SetID(id string) { - if e.Meta == nil { - e.Meta = mapstr.M{} - } - e.Meta["_id"] = id + _, _ = e.PutValue(metadataKeyPrefix+"_id", id) } +// GetValue gets a value from the event. If the key does not exist then an error +// is returned. +// +// Use `@timestamp` key for getting the event timestamp. +// Use `@metadata.*` keys for getting the event metadata fields. +// If `@metadata` key is used then `ErrMetadataAccess` is returned. func (e *Event) GetValue(key string) (interface{}, error) { - if key == timestampFieldKey { + if key == TimestampFieldKey { return e.Timestamp, nil - } else if subKey, ok := metadataKey(key); ok { - if subKey == "" || e.Meta == nil { - return e.Meta, nil + } + if key == MetadataFieldKey { + return nil, ErrMetadataAccess + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + return nil, mapstr.ErrKeyNotFound } return e.Meta.GetValue(subKey) } + + if e.Fields == nil { + return nil, mapstr.ErrKeyNotFound + } + return e.Fields.GetValue(key) } @@ -92,7 +119,7 @@ func (e *Event) Clone() *Event { // `DeepUpdateNoOverwrite` is a version of this function that does not // overwrite existing values. func (e *Event) DeepUpdate(d mapstr.M) { - e.deepUpdate(d, true) + e.deepUpdate(d, updateModeOverwrite) } // DeepUpdateNoOverwrite recursively copies the key-value pairs from `d` to various properties of the event. @@ -103,31 +130,34 @@ func (e *Event) DeepUpdate(d mapstr.M) { // via `DeepUpdateNoOverwrite`. // `DeepUpdate` is a version of this function that overwrites existing values. func (e *Event) DeepUpdateNoOverwrite(d mapstr.M) { - e.deepUpdate(d, false) + e.deepUpdate(d, updateModeNoOverwrite) } -func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { +func (e *Event) deepUpdate(d mapstr.M, mode updateMode) { if len(d) == 0 { return } // It's supported to update the timestamp using this function. // However, we must handle it separately since it's a separate field of the event. - timestampValue, timestampExists := d[timestampFieldKey] + timestampValue, timestampExists := d[TimestampFieldKey] if timestampExists { - if overwrite { - _ = e.setTimestamp(timestampValue) + if mode == updateModeOverwrite { + _, _ = e.setTimestamp(timestampValue) } // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, timestampFieldKey) + delete(d, TimestampFieldKey) + defer func() { + d[TimestampFieldKey] = timestampValue + }() } // It's supported to update the metadata using this function. // However, we must handle it separately since it's a separate field of the event. - metaValue, metaExists := d[metadataFieldKey] + metaValue, metaExists := d[MetadataFieldKey] if metaExists { var metaUpdate mapstr.M @@ -142,9 +172,10 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { if e.Meta == nil { e.Meta = mapstr.M{} } - if overwrite { + switch mode { + case updateModeOverwrite: e.Meta.DeepUpdate(metaUpdate) - } else { + case updateModeNoOverwrite: e.Meta.DeepUpdateNoOverwrite(metaUpdate) } } @@ -152,19 +183,12 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { // Temporary delete it from the update map, // so we can do `e.Fields.DeepUpdate(d)` or // `e.Fields.DeepUpdateNoOverwrite(d)` later - delete(d, metadataFieldKey) + delete(d, MetadataFieldKey) + defer func() { + d[MetadataFieldKey] = metaValue + }() } - // At the end we revert all changes we made to the update map - defer func() { - if timestampExists { - d[timestampFieldKey] = timestampValue - } - if metaExists { - d[metadataFieldKey] = metaValue - } - }() - if len(d) == 0 { return } @@ -173,90 +197,150 @@ func (e *Event) deepUpdate(d mapstr.M, overwrite bool) { e.Fields = mapstr.M{} } - if overwrite { + switch mode { + case updateModeOverwrite: e.Fields.DeepUpdate(d) - } else { + case updateModeNoOverwrite: e.Fields.DeepUpdateNoOverwrite(d) } } -func (e *Event) setTimestamp(v interface{}) error { +func (e *Event) setTimestamp(v interface{}) (interface{}, error) { + // to satisfy the PutValue interface, this function + // must return the overwritten value + prevValue := e.Timestamp + switch ts := v.(type) { case time.Time: e.Timestamp = ts + return prevValue, nil case common.Time: e.Timestamp = time.Time(ts) + return prevValue, nil default: - return errNoTimestamp + return nil, ErrValueNotTimestamp } - - return nil } +// Put associates the specified value with the specified key. If the event +// previously contained a mapping for the key, the old value is replaced and +// returned. The key can be expressed in dot-notation (e.g. x.y) to put a value +// into a nested map. +// +// If you need insert keys containing dots then you must use bracket notation +// to insert values (e.g. m[key] = value). +// +// Use `@timestamp` key for setting the event timestamp. +// Use `@metadata.*` keys for setting the event metadata fields. +// If `@metadata` key is used then `ErrAlterMetadataKey` is returned. func (e *Event) PutValue(key string, v interface{}) (interface{}, error) { - if key == timestampFieldKey { - err := e.setTimestamp(v) - return nil, err - } else if subKey, ok := metadataKey(key); ok { - if subKey == "" { - switch meta := v.(type) { - case mapstr.M: - e.Meta = meta - case map[string]interface{}: - e.Meta = meta - default: - return nil, errNoMapStr - } - } else if e.Meta == nil { + if key == TimestampFieldKey { + return e.setTimestamp(v) + } + if key == MetadataFieldKey { + return nil, ErrAlterMetadataKey + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { e.Meta = mapstr.M{} } + return e.Meta.Put(subKey, v) } + if e.Fields == nil { + e.Fields = mapstr.M{} + } + return e.Fields.Put(key, v) } +// Delete deletes the given key from the event. +// +// Use `@metadata.*` keys for deleting the event metadata fields. +// If `@metadata` key is used then `ErrAlterMetadataKey` is returned. +// If `@timestamp` key is used then `ErrDeleteTimestamp` is returned. func (e *Event) Delete(key string) error { - if subKey, ok := metadataKey(key); ok { - if subKey == "" { - e.Meta = nil - return nil - } + if key == TimestampFieldKey { + return ErrDeleteTimestamp + } + if key == MetadataFieldKey { + return ErrAlterMetadataKey + } + if subKey, ok := e.metadataSubKey(key); ok { if e.Meta == nil { - return nil + return mapstr.ErrKeyNotFound } return e.Meta.Delete(subKey) } + + if e.Fields == nil { + return mapstr.ErrKeyNotFound + } return e.Fields.Delete(key) } -func metadataKey(key string) (string, bool) { - if !strings.HasPrefix(key, metadataFieldKey) { +func (e *Event) metadataSubKey(key string) (string, bool) { + if !strings.HasPrefix(key, metadataKeyPrefix) { return "", false } - subKey := key[len(metadataFieldKey):] + subKey := key[metadataKeyOffset:] if subKey == "" { - return "", true - } - if subKey[0] == '.' { - return subKey[1:], true + return "", false } - return "", false + return subKey, true } // SetErrorWithOption sets the event error field with the message when the addErrKey is set to true. // If you want to include the data and field you can pass them as parameters and will be appended into the // error as fields with the corresponding name. func (e *Event) SetErrorWithOption(message string, addErrKey bool, data string, field string) { - if addErrKey { - errorField := mapstr.M{"message": message, "type": "json"} - if data != "" { - errorField["data"] = data - } - if field != "" { - errorField["field"] = field + if !addErrKey { + return + } + + errorField := mapstr.M{"message": message, "type": "json"} + if data != "" { + errorField["data"] = data + } + if field != "" { + errorField["field"] = field + } + e.Fields[ErrorFieldKey] = errorField +} + +// String returns a string representation of the event. +func (e *Event) String() string { + m := mapstr.M{ + TimestampFieldKey: e.Timestamp, + MetadataFieldKey: mapstr.M{}, + } + if e.Meta != nil { + m[MetadataFieldKey] = e.Meta + } + m.DeepUpdate(e.Fields) + return m.String() +} + +// HasKey returns true if the key exist. If an error occurs then false is +// returned with a non-nil error. +func (e *Event) HasKey(key string) (bool, error) { + if key == TimestampFieldKey || key == MetadataFieldKey { + return true, nil + } + + if subKey, ok := e.metadataSubKey(key); ok { + if e.Meta == nil { + return false, nil } - e.Fields["error"] = errorField + return e.Meta.HasKey(subKey) } + + if e.Fields == nil { + return false, nil + } + + return e.Fields.HasKey(key) } diff --git a/libbeat/beat/event_test.go b/libbeat/beat/event_test.go index cd165a3c4593..24ffb87dfa3e 100644 --- a/libbeat/beat/event_test.go +++ b/libbeat/beat/event_test.go @@ -18,351 +18,569 @@ package beat import ( - "crypto/rand" "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent-libs/mapstr" ) -const ( - propSize = 1024 * 2014 * 10 -) - -var largeProp string - -func init() { - b := make([]byte, propSize) - _, _ = rand.Read(b) - largeProp = string(b) -} - -func newEmptyEvent() *Event { - return &Event{Fields: mapstr.M{}} -} - -func newEvent(e mapstr.M) *Event { - n := &mapstr.M{ - "Fields": mapstr.M{ - "large_prop": largeProp, - }, - } - n.DeepUpdate(e) - var ts time.Time - var meta mapstr.M - var fields mapstr.M - var private mapstr.M - - v, ex := (*n)["Timestamp"] - if ex { - ts = v.(time.Time) +func TestEvent(t *testing.T) { + metadataNestedNestedMap := mapstr.M{ + "metaLevel2Value": "metavalue3", } - v, ex = (*n)["Meta"] - if ex { - meta = v.(mapstr.M) + metadataNestedMap := mapstr.M{ + "metaLevel1Map": metadataNestedNestedMap, } - v, ex = (*n)["Fields"] - if ex { - fields = v.(mapstr.M) + + fieldsNestedNestedMap := mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", } - v, ex = (*n)["Private"] - if ex { - private = v.(mapstr.M) + fieldsNestedMap := mapstr.M{ + "fieldsLevel1Map": fieldsNestedNestedMap, } - return &Event{ - Timestamp: ts, - Meta: meta, - Fields: fields, - Private: private, + + metaUntouchedMap := mapstr.M{} + fieldsUntouchedMap := mapstr.M{} + + event := &Event{ + Timestamp: time.Now(), + Meta: mapstr.M{ + "a.b": "c", + "metaLevel0Map": metadataNestedMap, + "metaLevel0Value": "metavalue1", + // these keys should never be edited by the tests + // to verify that existing keys remain + "metaLevel0Value2": "untouched", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "a.b": "c", + "fieldsLevel0Map": fieldsNestedMap, + "fieldsLevel0Value": "fieldsvalue1", + // these keys should never be edited by the tests + // to verify that existing keys remain + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, } -} -func BenchmarkTestEventPutGetTimestamp(b *testing.B) { - evt := newEmptyEvent() - ts := time.Now() + t.Run("empty", func(t *testing.T) { + t.Run("Delete", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + err := event.Delete(metadataKeyPrefix + "some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + err = event.Delete("some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + }) - evt.PutValue("@timestamp", ts) + t.Run("HasKey", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + has, err := event.HasKey(metadataKeyPrefix + "some") + require.NoError(t, err) + require.False(t, has) + has, err = event.HasKey("some") + require.NoError(t, err) + require.False(t, has) + }) + }) - v, err := evt.GetValue("@timestamp") - if err != nil { - b.Fatal(err) - } + t.Run("GetValue", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + _, err := event.GetValue(metadataKeyPrefix + "some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + _, err = event.GetValue("some") + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + }) - assert.Equal(b, ts, v) - assert.Equal(b, ts, evt.Timestamp) + t.Run("PutValue", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + prev, err := event.PutValue(metadataKeyPrefix+"some", "value") + require.NoError(t, err) + require.Nil(t, prev) + prev, err = event.PutValue("some", "value") + require.NoError(t, err) + require.Nil(t, prev) + }) + }) - // The @timestamp is not written into Fields. - assert.Nil(b, evt.Fields["@timestamp"]) -} + t.Run("DeepUpdate", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + event.DeepUpdate(mapstr.M{ + MetadataFieldKey: mapstr.M{"key": "value"}, + "key": "value", + }) + }) + }) -func BenchmarkTestDeepUpdate(b *testing.B) { - ts := time.Now() - - cases := []struct { - name string - event *Event - update mapstr.M - overwrite bool - expected *Event - }{ - { - name: "does nothing if no update", - event: newEvent(mapstr.M{}), - update: mapstr.M{}, - expected: newEvent(mapstr.M{}), - }, - { - name: "updates timestamp", - event: newEvent(mapstr.M{}), - update: mapstr.M{ - timestampFieldKey: ts, - }, - overwrite: true, - expected: &Event{ - Timestamp: ts, - Fields: mapstr.M{ - "large_prop": largeProp, - }, + t.Run("String", func(t *testing.T) { + event := &Event{} + require.NotPanics(t, func() { + s := event.String() + require.Equal(t, `{"@metadata":{},"@timestamp":"0001-01-01T00:00:00Z"}`, s) + }) + }) + }) + + t.Run("Get", func(t *testing.T) { + cases := []struct { + name string + key string + exp interface{} + expErr error + }{ + { + name: TimestampFieldKey, + key: TimestampFieldKey, + exp: event.Timestamp, }, - }, - { - name: "does not overwrite timestamp", - event: newEvent(mapstr.M{ - "Timestamp": ts, - }), - update: mapstr.M{ - timestampFieldKey: time.Now().Add(time.Hour), - }, - overwrite: false, - expected: &Event{ - Timestamp: ts, - Fields: mapstr.M{ - "large_prop": largeProp, - }, + { + name: "no acess to metadata key", + key: MetadataFieldKey, + expErr: ErrMetadataAccess, }, - }, - { - name: "initializes metadata if nil", - event: newEvent(mapstr.M{}), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ - "first": "new", - "second": 42, - }, + { + name: "non-existing metadata sub-key", + key: metadataKeyPrefix + "none", + expErr: mapstr.ErrKeyNotFound, }, - expected: &Event{ - Meta: mapstr.M{ - "first": "new", - "second": 42, - }, - Fields: mapstr.M{ - "large_prop": largeProp, - }, + { + name: "a value type from metadata", + key: metadataKeyPrefix + "metaLevel0Value", + exp: "metavalue1", }, - }, - { - name: "updates metadata but does not overwrite", - event: newEvent(mapstr.M{ - "Meta": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ - "first": "new", - "second": 42, - }, + { + name: "a root-level dot-key from metadata", + key: metadataKeyPrefix + "a.b", + exp: "c", }, - overwrite: false, - expected: &Event{ - Meta: mapstr.M{ - "first": "initial", - "second": 42, - }, - Fields: mapstr.M{ - "large_prop": largeProp, - }, + { + name: "a nested map from metadata", + key: metadataKeyPrefix + "metaLevel0Map", + exp: metadataNestedMap, }, - }, - { - name: "updates metadata and overwrites", - event: newEvent(mapstr.M{ - "Meta": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - metadataFieldKey: mapstr.M{ - "first": "new", - "second": 42, - }, + { + name: "non-existing field key", + key: "none", + expErr: mapstr.ErrKeyNotFound, }, - overwrite: true, - expected: &Event{ - Meta: mapstr.M{ - "first": "new", - "second": 42, - }, - Fields: mapstr.M{ - "large_prop": largeProp, - }, + { + name: "a value type from fields", + key: "fieldsLevel0Value", + exp: "fieldsvalue1", }, - }, - { - name: "updates fields but does not overwrite", - event: newEvent(mapstr.M{ - "Fields": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - "first": "new", - "second": 42, + { + name: "a root-level dot-key from fields", + key: "a.b", + exp: "c", }, - overwrite: false, - expected: &Event{ - Fields: mapstr.M{ - "first": "initial", - "second": 42, - "large_prop": largeProp, - }, + { + name: "a nested map from fields", + key: "fieldsLevel0Map", + exp: fieldsNestedMap, }, - }, - { - name: "updates metadata and overwrites", - event: newEvent(mapstr.M{ - "Fields": mapstr.M{ - "first": "initial", - }, - }), - update: mapstr.M{ - "first": "new", - "second": 42, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + val, err := event.GetValue(tc.key) + if tc.expErr != nil { + require.Error(t, err) + require.Nil(t, val) + require.ErrorIs(t, err, tc.expErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.exp, val) + }) + } + }) + + t.Run("Delete", func(t *testing.T) { + cases := []struct { + name string + key string + exp interface{} + expErr error + }{ + { + name: TimestampFieldKey, + key: TimestampFieldKey, + expErr: ErrDeleteTimestamp, }, - overwrite: true, - expected: &Event{ - Fields: mapstr.M{ - "first": "new", - "second": 42, - "large_prop": largeProp, - }, + { + name: "no acess to metadata key", + key: MetadataFieldKey, + expErr: ErrAlterMetadataKey, }, - }, - { - name: "initializes fields if nil", - event: newEvent(mapstr.M{}), - update: mapstr.M{ - "first": "new", - "second": 42, - }, - expected: &Event{ - Fields: mapstr.M{ - "first": "new", - "second": 42, - "large_prop": largeProp, - }, + { + name: "non-existing metadata sub key", + key: metadataKeyPrefix + "none", + expErr: mapstr.ErrKeyNotFound, }, - }, - } - - for _, tc := range cases { - b.Run(tc.name, func(b *testing.B) { - tc.event.deepUpdate(tc.update, tc.overwrite) - assert.Equal(b, tc.expected.Timestamp, tc.event.Timestamp) - assert.Equal(b, tc.expected.Fields, tc.event.Fields) - assert.Equal(b, tc.expected.Meta, tc.event.Meta) - }) - } -} - -func BenchmarkTestEventMetadata(b *testing.B) { - const id = "123" - newMeta := func() mapstr.M { return mapstr.M{"_id": id} } - - b.Run("put", func(b *testing.B) { - evt := newEmptyEvent() - meta := newMeta() - - evt.PutValue("@metadata", meta) - - assert.Equal(b, meta, evt.Meta) - assert.Empty(b, evt.Fields) - }) - - b.Run("get", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - meta, err := evt.GetValue("@metadata") - - assert.NoError(b, err) - assert.Equal(b, evt.Meta, meta) + { + name: "a value type from metadata", + key: metadataKeyPrefix + "metaLevel0Value", + exp: "metavalue1", + }, + { + name: "a root-level dot-key from metadata", + key: metadataKeyPrefix + "a.b", + exp: "c", + }, + { + name: "a nested map from metadata", + key: metadataKeyPrefix + "metaLevel0Map", + exp: metadataNestedMap, + }, + { + name: "non-existing field key", + key: "none", + expErr: mapstr.ErrKeyNotFound, + }, + { + name: "a value type from fields", + key: "fieldsLevel0Value", + exp: "fieldsvalue1", + }, + { + name: "a root-level dot-key from fields", + key: "a.b", + exp: "c", + }, + { + name: "a nested map from fields", + key: "fieldsLevel0Map", + exp: fieldsNestedMap, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + event := event.Clone() + err := event.Delete(tc.key) + if tc.expErr != nil { + require.Error(t, err) + require.ErrorIs(t, err, tc.expErr) + return + } + require.NoError(t, err) + _, err = event.GetValue(tc.key) + require.Error(t, err) + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }) + } }) - b.Run("put sub-key", func(b *testing.B) { - evt := newEmptyEvent() - - evt.PutValue("@metadata._id", id) - - assert.Equal(b, newMeta(), evt.Meta) - assert.Empty(b, evt.Fields) - }) + t.Run("PutValue", func(t *testing.T) { + newTs := time.Now().Add(time.Hour) + cases := []struct { + name string + key string + val interface{} + expPrev interface{} + expErr error + }{ + { + name: "timestamp", + key: TimestampFieldKey, + val: newTs, + expPrev: event.Timestamp, + }, + { + name: "incorrect type for timestamp", + key: TimestampFieldKey, + val: "wrong", + expErr: ErrValueNotTimestamp, + }, + { + name: "no acess to metadata key", + key: MetadataFieldKey, + expErr: ErrAlterMetadataKey, + }, + { + name: "non-existing metadata key", + key: metadataKeyPrefix + "none", + expPrev: nil, + }, + { + name: "a value type from metadata", + key: metadataKeyPrefix + "metaLevel0Value", + val: "some", + expPrev: "metavalue1", + }, + { + name: "a root-level dot-key from metadata", + key: metadataKeyPrefix + "a.b", + val: "d", + expPrev: "c", + }, + { + name: "a nested map from metadata", + key: metadataKeyPrefix + "metaLevel0Map", + val: "some", + expPrev: metadataNestedMap, + }, + { + name: "non-existing field key", + key: "none", + val: "some", + expPrev: nil, + }, + { + name: "a value type from fields", + key: "fieldsLevel0Value", + val: "some", + expPrev: "fieldsvalue1", + }, + { + name: "a root-level dot-key from fields", + key: "a.b", + val: "d", + expPrev: "c", + }, + { + name: "a nested map from fields", + key: "fieldsLevel0Map", + val: "some", + expPrev: fieldsNestedMap, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + event := event.Clone() + prevVal, err := event.PutValue(tc.key, tc.val) + if tc.expErr != nil { + require.Error(t, err) + require.ErrorIs(t, err, tc.expErr) + require.Nil(t, prevVal) + return + } + require.NoError(t, err) + require.Equal(t, tc.expPrev, prevVal) + actual, err := event.GetValue(tc.key) + require.NoError(t, err) + require.Equal(t, tc.val, actual) + has, err := event.HasKey(tc.key) + require.NoError(t, err) + require.True(t, has) + }) + } + + t.Run("type conflict", func(t *testing.T) { + event := &Event{ + Meta: mapstr.M{ + "a": 9, + "c": 10, + }, + Fields: mapstr.M{ + "a": 9, + "c": 10, + }, + } + + _, err := event.PutValue("a.c", 10) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + _, err = event.PutValue("a.value", 9) + require.Error(t, err) + require.Equal(t, "expected map but type is int", err.Error()) + }) - b.Run("get sub-key", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() + t.Run("hierarchy", func(t *testing.T) { + event := &Event{ + Fields: mapstr.M{ + "a.b": 1, + }, + } + err := event.Delete("a.b") + require.NoError(t, err) + + prev, err := event.PutValue("a.b.c", 1) + require.NoError(t, err) + require.Nil(t, prev) + + expFields := mapstr.M{ + "a": mapstr.M{ + "b": mapstr.M{ + "c": 1, + }, + }, + } - v, err := evt.GetValue("@metadata._id") + require.Equal(t, expFields, event.Fields) + }) - assert.NoError(b, err) - assert.Equal(b, id, v) + t.Run("SetID", func(t *testing.T) { + event := &Event{} + event.SetID("unique") + require.Equal(t, "unique", event.Meta["_id"]) + }) }) - b.Run("delete", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - err := evt.Delete("@metadata") - - assert.NoError(b, err) - assert.Nil(b, evt.Meta) + t.Run("SetErrorWithOption", func(t *testing.T) { + cloned := event.Clone() + cloned.SetErrorWithOption("message", false, "data", "field") + require.Equal(t, event, cloned) + expEvent := cloned.Clone() + expEvent.Fields[ErrorFieldKey] = mapstr.M{ + "message": "message", + "field": "field", + "data": "data", + "type": "json", + } + cloned.SetErrorWithOption("message", true, "data", "field") + require.Equal(t, expEvent, cloned) }) - b.Run("delete sub-key", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() - - err := evt.Delete("@metadata._id") - - assert.NoError(b, err) - assert.Empty(b, evt.Meta) - }) + t.Run("DeepUpdate", func(t *testing.T) { + newTs := time.Now().Add(time.Hour) + update := map[string]interface{}{ + TimestampFieldKey: newTs, + MetadataFieldKey: map[string]interface{}{ + "metaLevel0Map": mapstr.M{ // mix types on purpose, should support both + "metaLevel1Map": map[string]interface{}{ + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metareplaced1", + "new2": "newmetavalue2", + }, + "fieldsLevel0Map": map[string]interface{}{ + "fieldsLevel1Map": mapstr.M{ + "new3": "newfieldsvalue1", + }, + "newmap": map[string]interface{}{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsreplaced1", + } + + t.Run("empty", func(t *testing.T) { + cloned := event.Clone() + cloned.DeepUpdate(nil) + require.Equal(t, event.Meta, cloned.Meta) + require.Equal(t, event.Fields, cloned.Fields) + }) - b.Run("setID", func(b *testing.B) { - evt := newEmptyEvent() + t.Run("overwrite", func(t *testing.T) { + event := event.Clone() + event.DeepUpdate(update) - evt.SetID(id) + expEvent := &Event{ + Timestamp: newTs, + Meta: mapstr.M{ + "a.b": "c", + "metaLevel0Map": mapstr.M{ + "metaLevel1Map": mapstr.M{ + "metaLevel2Value": "metavalue3", + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metareplaced1", + "metaLevel0Value2": "untouched", + "new2": "newmetavalue2", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "a.b": "c", + "fieldsLevel0Map": mapstr.M{ + "fieldsLevel1Map": mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", + "new3": "newfieldsvalue1", + }, + "newmap": mapstr.M{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsreplaced1", + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } - assert.Equal(b, newMeta(), evt.Meta) - }) + require.Equal(t, expEvent.Timestamp, event.Timestamp) + require.Equal(t, expEvent.Meta, event.Meta) + require.Equal(t, expEvent.Fields, event.Fields) + }) - b.Run("put non-metadata", func(b *testing.B) { - evt := newEmptyEvent() + t.Run("no overwrite", func(t *testing.T) { + cloned := event.Clone() + cloned.DeepUpdateNoOverwrite(update) - evt.PutValue("@metadataSpecial", id) + expEvent := &Event{ + // should have the original/non-overwritten timestamp value + Timestamp: event.Timestamp, + Meta: mapstr.M{ + "a.b": "c", + "metaLevel0Map": mapstr.M{ + "metaLevel1Map": mapstr.M{ + "metaLevel2Value": "metavalue3", + "new1": "newmetavalue1", + }, + }, + "metaLevel0Value": "metavalue1", + "metaLevel0Value2": "untouched", + "new2": "newmetavalue2", + "metaUntouchedMap": metaUntouchedMap, + }, + Fields: mapstr.M{ + "a.b": "c", + "fieldsLevel0Map": mapstr.M{ + "fieldsLevel1Map": mapstr.M{ + "fieldsLevel2Value": "fieldsvalue3", + "new3": "newfieldsvalue1", + }, + "newmap": mapstr.M{ + "new4": "newfieldsvalue2", + }, + }, + "fieldsLevel0Value": "fieldsvalue1", + "fieldsLevel0Value2": "untouched", + "fieldsUntouchedMap": fieldsUntouchedMap, + }, + } - assert.Equal(b, mapstr.M{"@metadataSpecial": id}, evt.Fields) + require.Equal(t, expEvent.Timestamp, cloned.Timestamp) + require.Equal(t, expEvent.Meta, cloned.Meta) + require.Equal(t, expEvent.Fields, cloned.Fields) + }) }) - b.Run("delete non-metadata", func(b *testing.B) { - evt := newEmptyEvent() - evt.Meta = newMeta() + t.Run("String", func(t *testing.T) { + ts := time.Now().Add(time.Hour) + event := &Event{ + Timestamp: ts, + Meta: mapstr.M{ + "metakey": "metavalue", + }, + Fields: mapstr.M{ + "key": "value", + }, + } - err := evt.Delete("@metadataSpecial") + exp := mapstr.M{ + TimestampFieldKey: ts, + MetadataFieldKey: mapstr.M{ + "metakey": "metavalue", + }, + "key": "value", + } - assert.Error(b, err) - assert.Equal(b, newMeta(), evt.Meta) + require.Equal(t, exp.String(), event.String()) }) } diff --git a/libbeat/processors/add_id/add_id_test.go b/libbeat/processors/add_id/add_id_test.go index 18effb85205b..e3c5c410bf17 100644 --- a/libbeat/processors/add_id/add_id_test.go +++ b/libbeat/processors/add_id/add_id_test.go @@ -61,8 +61,8 @@ func TestNonDefaultTargetField(t *testing.T) { assert.NotEmpty(t, v) v, err = newEvent.GetValue("@metadata._id") - assert.NoError(t, err) - assert.Empty(t, v) + assert.Error(t, err) + assert.ErrorIs(t, err, mapstr.ErrKeyNotFound) } func TestNonDefaultMetadataTarget(t *testing.T) { From 670b682eb78dcbcda0bf43a98eb8f58f9d7af18f Mon Sep 17 00:00:00 2001 From: Michael Katsoulis Date: Tue, 24 Oct 2023 11:23:25 +0300 Subject: [PATCH 24/50] Use Merge with AppendValues option when merging default config with hints generated config (#36857) * Use Merge with AppendValues option when merging default config with hints generated config * Update filebeat/autodiscover/builder/hints/logs.go Co-authored-by: Andrew Gizas * Update changelog --------- Co-authored-by: Andrew Gizas --- CHANGELOG.next.asciidoc | 1 + filebeat/autodiscover/builder/hints/logs.go | 3 +- .../autodiscover/builder/hints/logs_test.go | 73 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7fbe4c0e480d..a8b019f0863b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -112,6 +112,7 @@ is collected by it. - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557] - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395] - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772] +- Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857] *Heartbeat* diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 72d8e4a63dd2..52f8c91672a0 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -139,7 +139,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log) } // Merge config template with the configs from the annotations - if err := config.Merge(tempCfg); err != nil { + // AppendValues option is used to append arrays from annotations to existing arrays while merging + if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil { logp.Debug("hints.builder", "config merge failed with error: %v", err) continue } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index a8bba01d52bd..cd2b72367712 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -43,6 +43,24 @@ func TestGenerateHints(t *testing.T) { }, }) + customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{ + "default_config": map[string]interface{}{ + "type": "container", + "paths": []string{ + "/var/lib/docker/containers/${data.container.id}/*-json.log", + }, + "close_timeout": "true", + "processors": []interface{}{ + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []string{"web"}, + "target": "environment", + }, + }, + }, + }, + }) + defaultCfg := conf.NewConfig() defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{ @@ -389,6 +407,61 @@ func TestGenerateHints(t *testing.T) { }, }, }, + { + msg: "Processors in hints must be appended in the processors of the default config", + config: customProcessorCfg, + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": mapstr.M{ + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + }, + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + "hints": mapstr.M{ + "logs": mapstr.M{ + "processors": mapstr.M{ + "1": mapstr.M{ + "dissect": mapstr.M{ + "tokenizer": "%{key1} %{key2}", + }, + }, + "drop_event": mapstr.M{}, + }, + }, + }, + }, + len: 1, + result: []mapstr.M{ + { + "type": "container", + "paths": []interface{}{ + "/var/lib/docker/containers/abc/*-json.log", + }, + "close_timeout": "true", + "processors": []interface{}{ + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []interface{}{"web"}, + "target": "environment", + }, + }, + map[string]interface{}{ + "dissect": map[string]interface{}{ + "tokenizer": "%{key1} %{key2}", + }, + }, + map[string]interface{}{ + "drop_event": nil, + }, + }, + }, + }, + }, { msg: "Hint with module should attach input to its filesets", config: customCfg, From effe78ceac5b786a9e7260b4188f4041af549146 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Tue, 24 Oct 2023 20:52:48 +1030 Subject: [PATCH 25/50] x-pack/filebeat/input/{cel,httpjson}: elide unneeded retryable HTTP client construction (#36916) --- CHANGELOG-developer.next.asciidoc | 1 + x-pack/filebeat/input/cel/input.go | 22 +++++----- x-pack/filebeat/input/httpjson/input.go | 53 +++++++++++++------------ 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index c3414579d70b..7fefb8d14c29 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -176,6 +176,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Make Filebeat HTTPJSON input process responses sequentially. {pull}36493[36493] - Add initial infrastructure for a caching enrichment processor. {pull}36619[36619] - Add file-backed cache for cache enrichment processor. {pull}36686[36686] {pull}36696[36696] +- Elide retryable HTTP client construction in Filebeat HTTPJSON and CEL inputs if not needed. {pull}36916[36916] ==== Deprecated diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 499ae97f7fac..03d716e0c6c8 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -721,25 +721,27 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, c.CheckRedirect = checkRedirect(cfg.Resource, log) - client := &retryablehttp.Client{ - HTTPClient: c, - Logger: newRetryLog(log), - RetryWaitMin: cfg.Resource.Retry.getWaitMin(), - RetryWaitMax: cfg.Resource.Retry.getWaitMax(), - RetryMax: cfg.Resource.Retry.getMaxAttempts(), - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, + if cfg.Resource.Retry.getMaxAttempts() > 1 { + c = (&retryablehttp.Client{ + HTTPClient: c, + Logger: newRetryLog(log), + RetryWaitMin: cfg.Resource.Retry.getWaitMin(), + RetryWaitMax: cfg.Resource.Retry.getWaitMax(), + RetryMax: cfg.Resource.Retry.getMaxAttempts(), + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + }).StandardClient() } if cfg.Auth.OAuth2.isEnabled() { - authClient, err := cfg.Auth.OAuth2.client(ctx, client.StandardClient()) + authClient, err := cfg.Auth.OAuth2.client(ctx, c) if err != nil { return nil, err } return authClient, nil } - return client.StandardClient(), nil + return c, nil } func wantClient(cfg config) bool { diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 592f9bce86d2..928c056d2d39 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -192,37 +192,38 @@ func sanitizeFileName(name string) string { } func newHTTPClient(ctx context.Context, config config, log *logp.Logger, reg *monitoring.Registry) (*httpClient, error) { - // Make retryable HTTP client - netHTTPClient, err := newNetHTTPClient(ctx, config.Request, log, reg) + client, err := newNetHTTPClient(ctx, config.Request, log, reg) if err != nil { return nil, err } - client := &retryablehttp.Client{ - HTTPClient: netHTTPClient, - Logger: newRetryLogger(log), - RetryWaitMin: config.Request.Retry.getWaitMin(), - RetryWaitMax: config.Request.Retry.getWaitMax(), - RetryMax: config.Request.Retry.getMaxAttempts(), - CheckRetry: retryablehttp.DefaultRetryPolicy, - Backoff: retryablehttp.DefaultBackoff, + if config.Request.Retry.getMaxAttempts() > 1 { + // Make retryable HTTP client if needed. + client = (&retryablehttp.Client{ + HTTPClient: client, + Logger: newRetryLogger(log), + RetryWaitMin: config.Request.Retry.getWaitMin(), + RetryWaitMax: config.Request.Retry.getWaitMax(), + RetryMax: config.Request.Retry.getMaxAttempts(), + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + }).StandardClient() } limiter := newRateLimiterFromConfig(config.Request.RateLimit, log) if config.Auth.OAuth2.isEnabled() { - authClient, err := config.Auth.OAuth2.client(ctx, client.StandardClient()) + authClient, err := config.Auth.OAuth2.client(ctx, client) if err != nil { return nil, err } return &httpClient{client: authClient, limiter: limiter}, nil } - return &httpClient{client: client.StandardClient(), limiter: limiter}, nil + return &httpClient{client: client, limiter: limiter}, nil } func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger, reg *monitoring.Registry) (*http.Client, error) { - // Make retryable HTTP client netHTTPClient, err := cfg.Transport.Client(clientOptions(cfg.URL.URL, cfg.KeepAlive.settings())...) if err != nil { return nil, err @@ -255,8 +256,7 @@ func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger, } func newChainHTTPClient(ctx context.Context, authCfg *authConfig, requestCfg *requestConfig, log *logp.Logger, reg *monitoring.Registry, p ...*Policy) (*httpClient, error) { - // Make retryable HTTP client - netHTTPClient, err := newNetHTTPClient(ctx, requestCfg, log, reg) + client, err := newNetHTTPClient(ctx, requestCfg, log, reg) if err != nil { return nil, err } @@ -268,27 +268,30 @@ func newChainHTTPClient(ctx context.Context, authCfg *authConfig, requestCfg *re retryPolicyFunc = retryablehttp.DefaultRetryPolicy } - client := &retryablehttp.Client{ - HTTPClient: netHTTPClient, - Logger: newRetryLogger(log), - RetryWaitMin: requestCfg.Retry.getWaitMin(), - RetryWaitMax: requestCfg.Retry.getWaitMax(), - RetryMax: requestCfg.Retry.getMaxAttempts(), - CheckRetry: retryPolicyFunc, - Backoff: retryablehttp.DefaultBackoff, + if requestCfg.Retry.getMaxAttempts() > 1 { + // Make retryable HTTP client if needed. + client = (&retryablehttp.Client{ + HTTPClient: client, + Logger: newRetryLogger(log), + RetryWaitMin: requestCfg.Retry.getWaitMin(), + RetryWaitMax: requestCfg.Retry.getWaitMax(), + RetryMax: requestCfg.Retry.getMaxAttempts(), + CheckRetry: retryPolicyFunc, + Backoff: retryablehttp.DefaultBackoff, + }).StandardClient() } limiter := newRateLimiterFromConfig(requestCfg.RateLimit, log) if authCfg != nil && authCfg.OAuth2.isEnabled() { - authClient, err := authCfg.OAuth2.client(ctx, client.StandardClient()) + authClient, err := authCfg.OAuth2.client(ctx, client) if err != nil { return nil, err } return &httpClient{client: authClient, limiter: limiter}, nil } - return &httpClient{client: client.StandardClient(), limiter: limiter}, nil + return &httpClient{client: client, limiter: limiter}, nil } // clientOption returns constructed client configuration options, including From a572a424c3d35d1a61f1ee3257b9dd6378225512 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Wed, 25 Oct 2023 08:21:40 +0200 Subject: [PATCH 26/50] Document ES output API usage (#36940) This commit documents the `_bulk` API usage by the Elasticsearch output and how different status codes are handled. Co-authored-by: Denis --- .../elasticsearch/docs/elasticsearch.asciidoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 5ea65c16dc4e..7e3e64b6cc5c 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -750,3 +750,17 @@ output.elasticsearch: non_indexable_policy.dead_letter_index: index: "my-dead-letter-index" ------------------------------------------------------------------------------ + +[[es-apis]] +==== Elasticsearch APIs +{beatname_uc} will use the `_bulk` API from {es}, the events are sent +in the order they arrive to the publishing pipeline, a single `_bulk` +request may contain events from different inputs/modules. Temporary +failures are re-tried. + +The status code for each event is checked and handled as: + +* `< 300`: The event is counted as `events.acked` +* `409` (Conflict): The event is counted as `events.duplicates` +* `429` (Too Many Requests): The event is counted as `events.toomany` +* `> 399 and < 500`: The `non_indexable_policy` is applied. \ No newline at end of file From bbf0111c1d50b13e9e2742758ab19e89562ad1a4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Wed, 25 Oct 2023 20:46:50 +1030 Subject: [PATCH 27/50] x-pack/filebeat/input/internal/httplog: fix incorrect append source (#36956) --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/internal/httplog/roundtripper.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a8b019f0863b..43427a7365bc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -113,6 +113,7 @@ is collected by it. - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395] - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772] - Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857] +- Fix handling of response errors in HTTPJSON and CEL request trace logging. {pull}36956[36956] *Heartbeat* diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index bbb76cb50452..78e872efa665 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -158,9 +158,9 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err switch len(errorsMessages) { case 0: case 1: - respParts = append(reqParts, zap.String("error.message", errorsMessages[0])) + respParts = append(respParts, zap.String("error.message", errorsMessages[0])) default: - respParts = append(reqParts, zap.Strings("error.message", errorsMessages)) + respParts = append(respParts, zap.Strings("error.message", errorsMessages)) } log.Debug("HTTP response", respParts...) From 5d6c308da027fd7702e60a8b64cdbe0b40755355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:53:16 +0200 Subject: [PATCH 28/50] [Heartbeat] Fix monitor duration wrapper (#36900) Fixes #36892. Monitor duration is not being calculated correctly, where start time is initialized after monitor execution and wrapping order is overriding retries event order. --- CHANGELOG.next.asciidoc | 1 + .../monitors/wrappers/summarizer/plugdrop.go | 4 + .../monitors/wrappers/summarizer/plugerr.go | 6 + .../wrappers/summarizer/plugmondur.go | 15 +- .../wrappers/summarizer/plugstatestat.go | 4 + .../monitors/wrappers/summarizer/plugurl.go | 2 + .../wrappers/summarizer/summarizer.go | 19 +- .../wrappers/summarizer/summarizer_test.go | 183 ++++++++++++++++++ 8 files changed, 227 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 43427a7365bc..4e9a5f0128e2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -119,6 +119,7 @@ is collected by it. - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842] +- Fix monitor duration calculation with retries. {pull}36900[36900] *Metricbeat* diff --git a/heartbeat/monitors/wrappers/summarizer/plugdrop.go b/heartbeat/monitors/wrappers/summarizer/plugdrop.go index fff6c143bf02..a4ddc61abe7d 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugdrop.go +++ b/heartbeat/monitors/wrappers/summarizer/plugdrop.go @@ -43,3 +43,7 @@ func (d DropBrowserExtraEvents) BeforeSummary(event *beat.Event) BeforeSummaryAc func (d DropBrowserExtraEvents) BeforeRetry() { // noop } + +func (d DropBrowserExtraEvents) BeforeEachEvent(event *beat.Event) { + // noop +} diff --git a/heartbeat/monitors/wrappers/summarizer/plugerr.go b/heartbeat/monitors/wrappers/summarizer/plugerr.go index 1010370f520c..83ab6de4f5ab 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugerr.go +++ b/heartbeat/monitors/wrappers/summarizer/plugerr.go @@ -46,6 +46,8 @@ func NewBrowserErrPlugin() *BrowserErrPlugin { } } +func (esp *BrowserErrPlugin) BeforeEachEvent(event *beat.Event) {} // noop + func (esp *BrowserErrPlugin) EachEvent(event *beat.Event, eventErr error) EachEventActions { // track these to determine if the journey // needs an error injected due to incompleteness @@ -127,6 +129,10 @@ func (esp *LightweightErrPlugin) BeforeRetry() { // noop } +func (esp *LightweightErrPlugin) BeforeEachEvent(event *beat.Event) { + // noop +} + // errToFieldVal reflects on the error and returns either an *ecserr.ECSErr if possible, and a look.Reason otherwise func errToFieldVal(eventErr error) (errVal interface{}) { var asECS *ecserr.ECSErr diff --git a/heartbeat/monitors/wrappers/summarizer/plugmondur.go b/heartbeat/monitors/wrappers/summarizer/plugmondur.go index f677e57693f8..d71cc96ff2c1 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugmondur.go +++ b/heartbeat/monitors/wrappers/summarizer/plugmondur.go @@ -31,12 +31,15 @@ type LightweightDurationPlugin struct { } func (lwdsp *LightweightDurationPlugin) EachEvent(event *beat.Event, _ error) EachEventActions { - // Effectively only runs once, on the first event + return 0 // noop +} + +func (lwdsp *LightweightDurationPlugin) BeforeEachEvent(event *beat.Event) { + // Effectively capture on the first event if lwdsp.startedAt == nil { now := time.Now() lwdsp.startedAt = &now } - return 0 } func (lwdsp *LightweightDurationPlugin) BeforeSummary(event *beat.Event) BeforeSummaryActions { @@ -44,7 +47,10 @@ func (lwdsp *LightweightDurationPlugin) BeforeSummary(event *beat.Event) BeforeS return 0 } -func (lwdsp *LightweightDurationPlugin) BeforeRetry() {} +func (lwdsp *LightweightDurationPlugin) BeforeRetry() { + // Reset event start time + lwdsp.startedAt = nil +} // BrowserDurationPlugin handles the logic for writing the `monitor.duration.us` field // for browser monitors. @@ -82,4 +88,5 @@ func (bwdsp *BrowserDurationPlugin) BeforeSummary(event *beat.Event) BeforeSumma return 0 } -func (bwdsp *BrowserDurationPlugin) BeforeRetry() {} +func (bwdsp *BrowserDurationPlugin) BeforeRetry() {} +func (bwdsp *BrowserDurationPlugin) BeforeEachEvent(event *beat.Event) {} // noop diff --git a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go index 4acfee4dc361..cf7e90af5f30 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go +++ b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go @@ -74,6 +74,8 @@ func (ssp *BrowserStateStatusPlugin) BeforeRetry() { ssp.cssp.BeforeRetry() } +func (ssp *BrowserStateStatusPlugin) BeforeEachEvent(event *beat.Event) {} //noop + // LightweightStateStatusPlugin encapsulates the writing of the primary fields used by the summary, // those being `state.*`, `status.*` , `event.type`, and `monitor.check_group` type LightweightStateStatusPlugin struct { @@ -113,6 +115,8 @@ func (ssp *LightweightStateStatusPlugin) BeforeRetry() { ssp.cssp.BeforeRetry() } +func (ssp *LightweightStateStatusPlugin) BeforeEachEvent(event *beat.Event) {} // noop + type commonSSP struct { js *jobsummary.JobSummary stateTracker *monitorstate.Tracker diff --git a/heartbeat/monitors/wrappers/summarizer/plugurl.go b/heartbeat/monitors/wrappers/summarizer/plugurl.go index dc4394aa42ad..e47463575a31 100644 --- a/heartbeat/monitors/wrappers/summarizer/plugurl.go +++ b/heartbeat/monitors/wrappers/summarizer/plugurl.go @@ -52,3 +52,5 @@ func (busp *BrowserURLPlugin) BeforeSummary(event *beat.Event) BeforeSummaryActi func (busp *BrowserURLPlugin) BeforeRetry() { busp.urlFields = nil } + +func (busp *BrowserURLPlugin) BeforeEachEvent(event *beat.Event) {} //noop diff --git a/heartbeat/monitors/wrappers/summarizer/summarizer.go b/heartbeat/monitors/wrappers/summarizer/summarizer.go index 9c3f1bd8abdf..ad0902d45af7 100644 --- a/heartbeat/monitors/wrappers/summarizer/summarizer.go +++ b/heartbeat/monitors/wrappers/summarizer/summarizer.go @@ -42,6 +42,12 @@ type Summarizer struct { startedAt time.Time } +func (s Summarizer) beforeEachEvent(event *beat.Event) { + for _, plugin := range s.plugins { + plugin.BeforeEachEvent(event) + } +} + // EachEventActions is a set of options using bitmasks to inform execution after the EachEvent callback type EachEventActions uint8 @@ -58,6 +64,9 @@ const RetryBeforeSummary = 1 // in one location. Prior to this code was strewn about a bit more and following it was // a bit trickier. type SummarizerPlugin interface { + // BeforeEachEvent is called on each event, and allows for the mutation of events + // before monitor execution + BeforeEachEvent(event *beat.Event) // EachEvent is called on each event, and allows for the mutation of events EachEvent(event *beat.Event, err error) EachEventActions // BeforeSummary is run on the final (summary) event for each monitor. @@ -106,6 +115,10 @@ func (s *Summarizer) setupPlugins() { // This adds the state and summary top level fields. func (s *Summarizer) Wrap(j jobs.Job) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { + + // call BeforeEachEvent for each plugin before running job + s.beforeEachEvent(event) + conts, eventErr := j(event) s.mtx.Lock() @@ -145,14 +158,14 @@ func (s *Summarizer) Wrap(j jobs.Job) jobs.Job { // kibana queries // 2. If the site error is very short 1s gives it a tiny bit of time to recover delayedRootJob := func(event *beat.Event) ([]jobs.Job, error) { + time.Sleep(s.retryDelay) for _, p := range s.plugins { p.BeforeRetry() } - time.Sleep(s.retryDelay) - return s.rootJob(event) + return s.Wrap(s.rootJob)(event) } - conts = []jobs.Job{delayedRootJob} + return []jobs.Job{delayedRootJob}, eventErr } } diff --git a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go index 2a94b3e6f596..e579a649c8ef 100644 --- a/heartbeat/monitors/wrappers/summarizer/summarizer_test.go +++ b/heartbeat/monitors/wrappers/summarizer/summarizer_test.go @@ -19,11 +19,13 @@ package summarizer import ( "fmt" + "sync" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/heartbeat/look" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers/monitorstate" @@ -219,3 +221,184 @@ func TestSummarizer(t *testing.T) { }) } } + +// Test wrapper plugin hook order. Guaranteed order for plugins to be called upon determines +// what data can be appended to the event at each stage through retries. With this guarantee, +// plugins just need to ascertain that their invariants apply through hook execution order +func TestSummarizerPluginOrder(t *testing.T) { + t.Parallel() + + // these tests use strings to describe sequences of events + tests := []struct { + name string + maxAttempts int + expectedOrder []string + }{ + { + "one attempt", + 1, + []string{"bee", "job", "ee", "bs"}, + }, + { + "two attempts", + 2, + []string{"bee", "job", "ee", "bs", "br", "bee", "job", "ee", "bs"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Monitor setup + tracker := monitorstate.NewTracker(monitorstate.NilStateLoader, false) + sf := stdfields.StdMonitorFields{ID: "testmon", Name: "testmon", Type: "http", MaxAttempts: uint16(tt.maxAttempts)} + + // Test locals + calls := make(chan string, 100) + mtx := sync.Mutex{} + appendCall := func(event string) { + mtx.Lock() + defer mtx.Unlock() + + // Append call to global chan + calls <- event + } + + // We simplify these to always down since hook order should not be + // determined by status + job := func(event *beat.Event) (j []jobs.Job, retErr error) { + + calls <- "job" + + event.Fields = mapstr.M{ + "monitor": mapstr.M{ + "id": "test", + "status": string(monitorstate.StatusDown), + }, + } + + return nil, fmt.Errorf("dummyerr") + } + + s := NewSummarizer(job, sf, tracker) + // Shorten retry delay to make tests run faster + s.retryDelay = 2 * time.Millisecond + // Add mock plugin + s.plugins = append(s.plugins, &MockPlugin{ + eachEvent: func(_ *beat.Event, _ error) { + appendCall("ee") + }, + beforeSummary: func(_ *beat.Event) { + appendCall("bs") + }, + beforeRetry: func() { + appendCall("br") + }, + beforeEachEvent: func(_ *beat.Event) { + appendCall("bee") + }, + }) + wrapped := s.Wrap(job) + + _, _ = jobs.ExecJobAndConts(t, wrapped) + + close(calls) + + // gather order + rcvdOrder := []string{} + for c := range calls { + rcvdOrder = append(rcvdOrder, c) + } + + require.Equal(t, tt.expectedOrder, rcvdOrder) + require.Len(t, rcvdOrder, len(tt.expectedOrder)) + }) + } +} + +func TestRetryLightweightMonitorDuration(t *testing.T) { + t.Parallel() + + // Monitor setup + tracker := monitorstate.NewTracker(monitorstate.NilStateLoader, false) + sf := stdfields.StdMonitorFields{ID: "testmon", Name: "testmon", Type: "http", MaxAttempts: uint16(2)} + + // We simplify these to always down + job := func(event *beat.Event) (j []jobs.Job, retErr error) { + + // some platforms don't have enough precision to track immediate monitors time + time.Sleep(100 * time.Millisecond) + + event.Fields = mapstr.M{ + "monitor": mapstr.M{ + "id": "test", + "status": string(monitorstate.StatusDown), + }, + } + + return nil, fmt.Errorf("dummyerr") + } + + var retryStart time.Time + + s := NewSummarizer(job, sf, tracker) + // Shorten retry delay to make tests run faster + s.retryDelay = 2 * time.Millisecond + // Add mock plugin + s.plugins = append(s.plugins, &MockPlugin{ + beforeRetry: func() { + retryStart = time.Now() + }, + eachEvent: func(_ *beat.Event, _ error) {}, + beforeSummary: func(_ *beat.Event) {}, + beforeEachEvent: func(_ *beat.Event) {}, + }) + wrapped := s.Wrap(job) + + events, _ := jobs.ExecJobAndConts(t, wrapped) + + retryElapsed := time.Since(retryStart) + require.False(t, retryStart.IsZero()) + var rcvdDuration interface{} + for _, event := range events { + summaryIface, _ := event.GetValue("summary") + summary := summaryIface.(*jobsummary.JobSummary) + + if summary.FinalAttempt { + rcvdDuration, _ = event.GetValue("monitor.duration.us") + } + } + require.Greater(t, rcvdDuration, int64(0)) + // Ensures monitor duration only takes into account the last attempt execution time + // by comparing it to the time spent after last retry started (retryElapsed) + require.GreaterOrEqual(t, look.RTTMS(retryElapsed), rcvdDuration) +} + +type MockPlugin struct { + eachEvent func(e *beat.Event, err error) + beforeSummary func(e *beat.Event) + beforeRetry func() + beforeEachEvent func(e *beat.Event) +} + +func (mp *MockPlugin) EachEvent(e *beat.Event, err error) EachEventActions { + mp.eachEvent(e, err) + + return 0 +} + +func (mp *MockPlugin) BeforeSummary(e *beat.Event) BeforeSummaryActions { + mp.beforeSummary(e) + + return 0 +} + +func (mp *MockPlugin) BeforeRetry() { + mp.beforeRetry() +} + +func (mp *MockPlugin) BeforeEachEvent(e *beat.Event) { + mp.beforeEachEvent(e) +} From 55df09fadeae98979aa012801b669c7fbef321dd Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:26:36 -0500 Subject: [PATCH 29/50] Add support for idle_connection_timeout to elasticsearch output (#36843) * Add support for idle_connection_timeout to elasticsearch output --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 5 +++++ filebeat/filebeat.reference.yml | 5 +++++ heartbeat/heartbeat.reference.yml | 5 +++++ .../_meta/config/output-elasticsearch.reference.yml.tmpl | 5 +++++ libbeat/outputs/elasticsearch/client.go | 1 + libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc | 6 ++++++ libbeat/outputs/elasticsearch/elasticsearch.go | 5 ++++- metricbeat/metricbeat.reference.yml | 5 +++++ packetbeat/packetbeat.reference.yml | 5 +++++ winlogbeat/winlogbeat.reference.yml | 5 +++++ x-pack/auditbeat/auditbeat.reference.yml | 5 +++++ x-pack/filebeat/filebeat.reference.yml | 5 +++++ x-pack/functionbeat/functionbeat.reference.yml | 5 +++++ x-pack/heartbeat/heartbeat.reference.yml | 5 +++++ x-pack/metricbeat/metricbeat.reference.yml | 5 +++++ x-pack/osquerybeat/osquerybeat.reference.yml | 5 +++++ x-pack/packetbeat/packetbeat.reference.yml | 5 +++++ x-pack/winlogbeat/winlogbeat.reference.yml | 5 +++++ 19 files changed, 87 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4e9a5f0128e2..a59b65297c28 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -173,6 +173,7 @@ is collected by it. - allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788] - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments +- elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index d4214eaf604b..a3a36dde753f 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -522,6 +522,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 45aff60ce236..f49390485aca 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1618,6 +1618,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e8b74f8c075e..fe6a72cd4746 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl index 4acd341da01e..d6ebe03ada5d 100644 --- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl +++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl @@ -81,6 +81,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index c80e95ebc907..b485807776e7 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -102,6 +102,7 @@ func NewClient( CompressionLevel: s.CompressionLevel, EscapeHTML: s.EscapeHTML, Transport: s.Transport, + IdleConnTimeout: s.IdleConnTimeout, }) if err != nil { return nil, err diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 7e3e64b6cc5c..6af56ac42db8 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -689,6 +689,12 @@ default is `1s`. The maximum number of seconds to wait before attempting to connect to Elasticsearch after a network error. The default is `60s`. +===== `idle_connection_timeout` + +The maximum amount of time an idle connection will remain idle before closing itself. +Zero means no limit. The format is a Go language duration (example 60s is 60 seconds). +The default is 0. + ===== `timeout` The http request timeout in seconds for the Elasticsearch request. The default is 90. diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index f7e388539243..145b5b65c2ea 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -41,7 +41,9 @@ func makeES( ) (outputs.Group, error) { log := logp.NewLogger(logSelector) if !cfg.HasField("bulk_max_size") { - _ = cfg.SetInt("bulk_max_size", -1, defaultBulkSize) + if err := cfg.SetInt("bulk_max_size", -1, defaultBulkSize); err != nil { + return outputs.Fail(err) + } } index, pipeline, err := buildSelectors(im, beat, cfg) @@ -105,6 +107,7 @@ func makeES( Observer: observer, EscapeHTML: esConfig.EscapeHTML, Transport: esConfig.Transport, + IdleConnTimeout: esConfig.Transport.IdleConnTimeout, }, Index: index, Pipeline: pipeline, diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index fc79ddb514c9..8508d7d6a700 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1357,6 +1357,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index cc05f7b52128..d31a7b901d9d 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 110370957cf7..bdc366d09024 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -404,6 +404,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 6d9a71ca99cc..09b343d8fe28 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -578,6 +578,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index bff96ef19973..90cbc52d8c22 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3988,6 +3988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index d3a2231a43ef..b7b075f26120 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -646,6 +646,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index e8b74f8c075e..fe6a72cd4746 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -614,6 +614,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 436693bdfbc7..bfb13acc660f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1918,6 +1918,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index f17d16e28b8a..6103f48c4280 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -365,6 +365,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index cc05f7b52128..d31a7b901d9d 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -988,6 +988,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index eec0bca80779..f3ff654bca24 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -406,6 +406,11 @@ output.elasticsearch: # Elasticsearch after a network error. The default is 60s. #backoff.max: 60s + # The maximum amount of time an idle connection will remain idle + # before closing itself. Zero means use the default of 60s. The + # format is a Go language duration (example 60s is 60 seconds). + # idle_connection_timeout: 60s + # Configure HTTP request timeout before failing a request to Elasticsearch. #timeout: 90 From d8a1377371141e7896827d6e4ae952cd6502a361 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:12:48 +1030 Subject: [PATCH 30/50] mod: update version of github.com/elastic/go-libaudit to v2.4.0 (#36964) --- CHANGELOG.next.asciidoc | 2 ++ NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a59b65297c28..aecead61b013 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -251,6 +251,8 @@ is collected by it. *Auditbeat* +- Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964] + *Libbeat* *Heartbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 7061208f1ef6..0d59be5ef12d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13659,11 +13659,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearc -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-libaudit/v2 -Version: v2.3.3 +Version: v2.4.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.3.3/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-libaudit/v2@v2.4.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 227b7659e633..037291b36cd7 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.3.5 github.com/elastic/elastic-agent-client/v7 v7.4.0 github.com/elastic/go-concert v0.2.0 - github.com/elastic/go-libaudit/v2 v2.3.3 + github.com/elastic/go-libaudit/v2 v2.4.0 github.com/elastic/go-licenser v0.4.1 github.com/elastic/go-lookslike v0.3.0 github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f diff --git a/go.sum b/go.sum index 6ce895072b50..8d1a2265e373 100644 --- a/go.sum +++ b/go.sum @@ -670,8 +670,8 @@ github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa2 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38= github.com/elastic/go-elasticsearch/v8 v8.10.0 h1:ALg3DMxSrx07YmeMNcfPf7cFh1Ep2+Qa19EOXTbwr2k= github.com/elastic/go-elasticsearch/v8 v8.10.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= -github.com/elastic/go-libaudit/v2 v2.3.3 h1:PO+9/HDSn65UAyydkkoTf81QET14fWmocHFiGEX/E6M= -github.com/elastic/go-libaudit/v2 v2.3.3/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo= +github.com/elastic/go-libaudit/v2 v2.4.0 h1:PqaGnB+dncrdUXqzQMyJu/dGysAtk6m5V3GIBMY473I= +github.com/elastic/go-libaudit/v2 v2.4.0/go.mod h1:AjlnhinP+kKQuUJoXLVrqxBM8uyhQmkzoV6jjsCFP4Q= github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4= github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-lookslike v0.3.0 h1:HDI/DQ65V85ZqM7D/sbxcK2wFFnh3+7iFvBk2v2FTHs= @@ -2341,6 +2341,7 @@ golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 8982110e828f512db0985ec5737881824b9309ba Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 26 Oct 2023 12:45:19 +0200 Subject: [PATCH 31/50] Add benchmark for processors that create event backups (#36960) This will come handy once we have another PR with optimizations. Also, extended a test case in the append processor due to the lack of coverage. --- libbeat/processors/actions/append_test.go | 3 +- libbeat/processors/processor_test.go | 221 ++++++++++++++++++++++ 2 files changed, 223 insertions(+), 1 deletion(-) diff --git a/libbeat/processors/actions/append_test.go b/libbeat/processors/actions/append_test.go index 8cb8549b389b..ddc03d6f322a 100644 --- a/libbeat/processors/actions/append_test.go +++ b/libbeat/processors/actions/append_test.go @@ -273,6 +273,7 @@ func Test_appendProcessor_Run(t *testing.T) { logger: log, config: appendProcessorConfig{ Fields: []string{"field"}, + Values: []interface{}{"value3", "value4"}, TargetField: "target", }, }, @@ -281,7 +282,7 @@ func Test_appendProcessor_Run(t *testing.T) { Meta: mapstr.M{}, Fields: mapstr.M{ "field": "I'm being appended", - "target": []interface{}{"value1", "value2", "I'm being appended"}, + "target": []interface{}{"value1", "value2", "I'm being appended", "value3", "value4"}, }, }, }, diff --git a/libbeat/processors/processor_test.go b/libbeat/processors/processor_test.go index 41ed628fbfb9..91d122365e74 100644 --- a/libbeat/processors/processor_test.go +++ b/libbeat/processors/processor_test.go @@ -18,15 +18,23 @@ package processors_test import ( + "fmt" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" _ "github.com/elastic/beats/v7/libbeat/processors/actions" _ "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata" + _ "github.com/elastic/beats/v7/libbeat/processors/add_process_metadata" + _ "github.com/elastic/beats/v7/libbeat/processors/convert" + _ "github.com/elastic/beats/v7/libbeat/processors/decode_csv_fields" + _ "github.com/elastic/beats/v7/libbeat/processors/dissect" + _ "github.com/elastic/beats/v7/libbeat/processors/extract_array" + _ "github.com/elastic/beats/v7/libbeat/processors/urldecode" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -566,3 +574,216 @@ func TestDropMissingFields(t *testing.T) { assert.Equal(t, expectedEvent, processedEvent.Fields) } + +const ( + fieldCount = 20 + depth = 3 +) + +func BenchmarkEventBackups(b *testing.B) { + // listing all the processors that revert changes in case of an error + yml := []map[string]interface{}{ + { + "append": map[string]interface{}{ + "target_field": "append_target", + "values": []interface{}{"third", "fourth"}, + "fail_on_error": true, + }, + }, + { + "copy_fields": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "copy_from", + "to": "copy.to", + }, + }, + "fail_on_error": true, + }, + }, + { + "decode_base64_field": map[string]interface{}{ + "field": map[string]interface{}{ + "from": "base64_from", + "to": "base64_to", + }, + "fail_on_error": true, + }, + }, + { + "decompress_gzip_field": map[string]interface{}{ + "field": map[string]interface{}{ + "from": "gzip_from", + "to": "gzip_to", + }, + "fail_on_error": true, + }, + }, + { + "rename": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "rename_from", + "to": "rename.to", + }, + }, + "fail_on_error": true, + }, + }, + { + "replace": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "field": "replace_test", + "pattern": "to replace", + "replacement": "replaced", + }, + }, + "fail_on_error": true, + }, + }, + { + "truncate_fields": map[string]interface{}{ + "fields": []interface{}{"to_truncate"}, + "max_characters": 4, + "fail_on_error": true, + }, + }, + { + "convert": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "convert_from", + "to": "convert.to", + "type": "integer", + }, + }, + "fail_on_error": true, + }, + }, + { + "decode_csv_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "csv_from": "csv.to", + }, + "fail_on_error": true, + }, + }, + // it creates a backup unless `ignore_failure` is true + { + "dissect": map[string]interface{}{ + "tokenizer": "%{key1} %{key2}", + "field": "to_dissect", + }, + }, + { + "extract_array": map[string]interface{}{ + "field": "array_test", + "mappings": map[string]interface{}{ + "array_first": 0, + "array_second": 1, + }, + "fail_on_error": true, + }, + }, + { + "urldecode": map[string]interface{}{ + "fields": []map[string]interface{}{ + { + "from": "url_from", + "to": "url.to", + }, + }, + + "fail_on_error": true, + }, + }, + } + + processors := GetProcessors(b, yml) + event := &beat.Event{ + Timestamp: time.Now(), + Meta: mapstr.M{}, + Fields: mapstr.M{ + "append_target": []interface{}{"first", "second"}, + "copy_from": "to_copy", + "base64_from": "dmFsdWU=", + // "decompressed data" + "gzip_from": string([]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 74, 73, 77, 206, 207, 45, 40, 74, 45, 46, 78, 77, 81, 72, 73, 44, 73, 4, 4, 0, 0, 255, 255, 108, 158, 105, 19, 17, 0, 0, 0}), + "rename_from": "renamed_value", + "replace_test": "something to replace", + "to_truncate": "something very long", + "convert_from": "42", + "csv_from": "1,2,3,4", + "to_dissect": "some words", + "array_test": []string{"first", "second"}, + "url_from": "https%3A%2F%2Fwww.elastic.co%3Fsome", + }, + } + + expFields := mapstr.M{ + "append_target": []interface{}{"first", "second", "third", "fourth"}, + "copy_from": "to_copy", + "copy": mapstr.M{ + "to": "to_copy", + }, + "base64_from": "dmFsdWU=", + "base64_to": "value", + "gzip_from": string([]byte{31, 139, 8, 0, 0, 0, 0, 0, 0, 255, 74, 73, 77, 206, 207, 45, 40, 74, 45, 46, 78, 77, 81, 72, 73, 44, 73, 4, 4, 0, 0, 255, 255, 108, 158, 105, 19, 17, 0, 0, 0}), + "gzip_to": "decompressed data", + "rename": mapstr.M{"to": "renamed_value"}, + "replace_test": "something replaced", + "to_truncate": "some", + "convert_from": "42", + "convert": mapstr.M{"to": int32(42)}, + "csv_from": "1,2,3,4", + "csv": mapstr.M{"to": []string{"1", "2", "3", "4"}}, + "to_dissect": "some words", + "dissect": mapstr.M{ + "key1": "some", + "key2": "words", + }, + "array_test": []string{"first", "second"}, + "array_first": "first", + "array_second": "second", + "url_from": "https%3A%2F%2Fwww.elastic.co%3Fsome", + "url": mapstr.M{"to": "https://www.elastic.co?some"}, + } + + generateFields(b, event.Meta, fieldCount, depth) + generateFields(b, event.Fields, fieldCount, depth) + + var ( + result *beat.Event + clone *beat.Event + err error + ) + + b.Run("run processors that use backups", func(b *testing.B) { + for i := 0; i < b.N; i++ { + clone = event.Clone() // necessary for making and comparing changes + result, err = processors.Run(clone) + } + require.NoError(b, err) + require.NotNil(b, result) + }) + + require.Equal(b, fmt.Sprintf("%p", clone), fmt.Sprintf("%p", result), "should be the same event") + for key := range expFields { + require.Equal(b, expFields[key], clone.Fields[key], fmt.Sprintf("%s does not match", key)) + } +} + +func generateFields(t require.TestingT, m mapstr.M, count, nesting int) { + for i := 0; i < count; i++ { + var err error + if nesting == 0 { + _, err = m.Put(fmt.Sprintf("field-%d", i), fmt.Sprintf("value-%d", i)) + } else { + nested := mapstr.M{} + generateFields(t, nested, count, nesting-1) + _, err = m.Put(fmt.Sprintf("field-%d", i), nested) + } + require.NoError(t, err) + } +} From ee864b52d115b531e8e8983ba35cc786b05069b0 Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:24:23 -0500 Subject: [PATCH 32/50] Deprecate RSA2ELK Filebeat modules (#36887) - Add deprecation notices to RSA2ELK Filebeat modules. - Add doc page for migrating off of deprecated modules. --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/howto/howto.asciidoc | 2 ++ .../migrate-from-deprecated-module.asciidoc | 30 +++++++++++++++++++ filebeat/docs/modules/barracuda.asciidoc | 2 ++ filebeat/docs/modules/bluecoat.asciidoc | 2 ++ filebeat/docs/modules/cisco.asciidoc | 4 +++ filebeat/docs/modules/cylance.asciidoc | 2 ++ filebeat/docs/modules/f5.asciidoc | 2 ++ filebeat/docs/modules/fortinet.asciidoc | 6 ++++ filebeat/docs/modules/imperva.asciidoc | 2 ++ filebeat/docs/modules/infoblox.asciidoc | 2 ++ filebeat/docs/modules/juniper.asciidoc | 4 +++ filebeat/docs/modules/microsoft.asciidoc | 2 ++ filebeat/docs/modules/netscout.asciidoc | 2 ++ filebeat/docs/modules/proofpoint.asciidoc | 2 ++ filebeat/docs/modules/radware.asciidoc | 2 ++ filebeat/docs/modules/snort.asciidoc | 2 ++ filebeat/docs/modules/sonicwall.asciidoc | 2 ++ filebeat/docs/modules/sophos.asciidoc | 2 ++ filebeat/docs/modules/squid.asciidoc | 2 ++ filebeat/docs/modules/tomcat.asciidoc | 2 ++ filebeat/docs/modules/zscaler.asciidoc | 2 ++ .../module/barracuda/_meta/docs.asciidoc | 2 ++ .../module/bluecoat/_meta/docs.asciidoc | 2 ++ .../filebeat/module/cisco/_meta/docs.asciidoc | 4 +++ .../module/cylance/_meta/docs.asciidoc | 2 ++ x-pack/filebeat/module/f5/_meta/docs.asciidoc | 2 ++ .../module/fortinet/_meta/docs.asciidoc | 6 ++++ .../module/imperva/_meta/docs.asciidoc | 2 ++ .../module/infoblox/_meta/docs.asciidoc | 2 ++ .../module/juniper/_meta/docs.asciidoc | 4 +++ .../module/microsoft/_meta/docs.asciidoc | 2 ++ .../module/netscout/_meta/docs.asciidoc | 2 ++ .../module/proofpoint/_meta/docs.asciidoc | 2 ++ .../module/radware/_meta/docs.asciidoc | 2 ++ .../filebeat/module/snort/_meta/docs.asciidoc | 2 ++ .../module/sonicwall/_meta/docs.asciidoc | 2 ++ .../module/sophos/_meta/docs.asciidoc | 2 ++ .../filebeat/module/squid/_meta/docs.asciidoc | 2 ++ .../module/tomcat/_meta/docs.asciidoc | 2 ++ .../module/zscaler/_meta/docs.asciidoc | 2 ++ 41 files changed, 125 insertions(+) create mode 100644 filebeat/docs/howto/migrate-from-deprecated-module.asciidoc diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index aecead61b013..74a66ef3c23e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -302,6 +302,7 @@ is collected by it. *Filebeat* +- Deprecate rsa2elk Filebeat modules. {issue}36125[36125] {pull}36887[36887] *Heartbeat* diff --git a/filebeat/docs/howto/howto.asciidoc b/filebeat/docs/howto/howto.asciidoc index 7d18cd482251..7b9a3cffa030 100644 --- a/filebeat/docs/howto/howto.asciidoc +++ b/filebeat/docs/howto/howto.asciidoc @@ -16,6 +16,7 @@ Learn how to perform common {beatname_uc} configuration tasks. * <> * <> * <> +* <> -- @@ -46,4 +47,5 @@ include::{libbeat-dir}/yaml.asciidoc[] include::migrate-to-filestream.asciidoc[] +include::migrate-from-deprecated-module.asciidoc[] diff --git a/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc b/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc new file mode 100644 index 000000000000..fd163353f92b --- /dev/null +++ b/filebeat/docs/howto/migrate-from-deprecated-module.asciidoc @@ -0,0 +1,30 @@ +[[migrate-from-deprecated-module]] +== Migrating from a Deprecated Filebeat Module + +If a Filebeat module has been deprecated, there are a few options available for +a path forward: + +1. Migrate to an Elastic integration, if available. The deprecation notice will +link to an appropriate integration, if one exists. + +2. https://www.elastic.co/guide/en/fleet/current/migrate-beats-to-agent.html[Migrate to Elastic Agent] +for ingesting logs. If a specific integration for the vendor/product does not +exist, then one of the custom integrations can be used for ingesting events. A +https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html[custom pipeline] +may also be attached to the integration for further processing. + - https://docs.elastic.co/integrations/cel[CEL Custom API] - Collect events from an API using CEL (Common Expression Language) + - https://docs.elastic.co/integrations/httpjson[Custom API] - Collect events from an API using the HTTPJSON input + - https://docs.elastic.co/integrations/gcp_pubsub[Custom Google Pub/Sub] - Collect events from Google Pub/Sub topics + - https://docs.elastic.co/integrations/http_endpoint[Custom HTTP Endpoint] - Collect events from a listening HTTP port + - https://docs.elastic.co/integrations/journald[Custom Journald] - Collect events from journald + - https://docs.elastic.co/integrations/kafka_log[Custom Kafka] - Collect events from a Kafka topic + - https://docs.elastic.co/integrations/log[Custom Logs] - Collect events from files + - https://docs.elastic.co/integrations/tcp[Custom TCP] - Collect events from a listening TCP port + - https://docs.elastic.co/integrations/udp[Custom UDP] - Collect events from a listening UDP port + - https://docs.elastic.co/integrations/winlog[Custom Windows Event] - Collect events from a Windows Event Log channel + +3. Migrate to a different Filebeat module. In some cases, a Filebeat module may +be superseded by a new module. The deprecation notice will link to an appropriate +module, if one exists. + +4. Use a custom Filebeat input, processors, and ingest pipeline (if necessary). diff --git a/filebeat/docs/modules/barracuda.asciidoc b/filebeat/docs/modules/barracuda.asciidoc index 4b9f65fdfaf5..958c9a4444b0 100644 --- a/filebeat/docs/modules/barracuda.asciidoc +++ b/filebeat/docs/modules/barracuda.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Barracuda module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/barracuda[Barracuda Web Application Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/bluecoat.asciidoc b/filebeat/docs/modules/bluecoat.asciidoc index 843ef578aa30..c9dc391c6b46 100644 --- a/filebeat/docs/modules/bluecoat.asciidoc +++ b/filebeat/docs/modules/bluecoat.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Bluecoat module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index da25fb38c05d..ca094907b986 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -281,6 +281,8 @@ include::../include/timezone-support.asciidoc[] [float] ==== `nexus` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_nexus[Cisco Nexus] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "cisconxos" device revision 134. @@ -326,6 +328,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `meraki` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_meraki[Cisco Meraki] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "ciscomeraki" device revision 118. diff --git a/filebeat/docs/modules/cylance.asciidoc b/filebeat/docs/modules/cylance.asciidoc index ecb870528817..8eb9b356ce7c 100644 --- a/filebeat/docs/modules/cylance.asciidoc +++ b/filebeat/docs/modules/cylance.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Cylance module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/f5.asciidoc b/filebeat/docs/modules/f5.asciidoc index 366587eb6757..d8a5f50d784e 100644 --- a/filebeat/docs/modules/f5.asciidoc +++ b/filebeat/docs/modules/f5.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == F5 module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/f5_bigip[F5 BIG-IP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/fortinet.asciidoc b/filebeat/docs/modules/fortinet.asciidoc index 3149c062ae2a..f6575c7db65d 100644 --- a/filebeat/docs/modules/fortinet.asciidoc +++ b/filebeat/docs/modules/fortinet.asciidoc @@ -81,6 +81,8 @@ events. Defaults to `[fortinet-firewall, forwarded]`. [float] ==== `clientendpoint` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_forticlient[Fortinet FortiClient Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "forticlientendpoint" device revision 0. @@ -126,6 +128,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimail` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimail[Fortinet FortiMail] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetfortimail" device revision 131. @@ -171,6 +175,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimanager` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimanager[Fortinet FortiManager Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetmgr" device revision 134. diff --git a/filebeat/docs/modules/imperva.asciidoc b/filebeat/docs/modules/imperva.asciidoc index 085c44e126b1..efe2a0bcdab0 100644 --- a/filebeat/docs/modules/imperva.asciidoc +++ b/filebeat/docs/modules/imperva.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Imperva module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/infoblox.asciidoc b/filebeat/docs/modules/infoblox.asciidoc index 50539d2f9bd3..2cc018bd20be 100644 --- a/filebeat/docs/modules/infoblox.asciidoc +++ b/filebeat/docs/modules/infoblox.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Infoblox module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/infoblox_nios[Infoblox NIOS] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/juniper.asciidoc b/filebeat/docs/modules/juniper.asciidoc index 274c559fae2c..dd445e809a7b 100644 --- a/filebeat/docs/modules/juniper.asciidoc +++ b/filebeat/docs/modules/juniper.asciidoc @@ -142,6 +142,8 @@ This is a list of JunOS fields that are mapped to ECS. [float] ==== `junos` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/juniper_srx[Juniper SRX] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "junosrouter" device revision 134. @@ -187,6 +189,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `netscreen` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. See <> for migration options."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "netscreen" device revision 134. diff --git a/filebeat/docs/modules/microsoft.asciidoc b/filebeat/docs/modules/microsoft.asciidoc index 2772f12d3f13..4e5ff72b30b1 100644 --- a/filebeat/docs/modules/microsoft.asciidoc +++ b/filebeat/docs/modules/microsoft.asciidoc @@ -224,6 +224,8 @@ And for all other Defender ATP event types, go to Host -> Events. [float] ==== `dhcp` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/microsoft_dhcp[Microsoft DHCP] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "msdhcp" device revision 99. diff --git a/filebeat/docs/modules/netscout.asciidoc b/filebeat/docs/modules/netscout.asciidoc index 552153d1c67a..2a9bfe34f679 100644 --- a/filebeat/docs/modules/netscout.asciidoc +++ b/filebeat/docs/modules/netscout.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Netscout module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] This is a module for receiving Arbor Peakflow SP logs over Syslog or a file. diff --git a/filebeat/docs/modules/proofpoint.asciidoc b/filebeat/docs/modules/proofpoint.asciidoc index 5f58e6764f3b..cb420ae1e10b 100644 --- a/filebeat/docs/modules/proofpoint.asciidoc +++ b/filebeat/docs/modules/proofpoint.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Proofpoint module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/proofpoint_tap[Proofpoint TAP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/radware.asciidoc b/filebeat/docs/modules/radware.asciidoc index 23ad0b7a1798..1ebab4928e42 100644 --- a/filebeat/docs/modules/radware.asciidoc +++ b/filebeat/docs/modules/radware.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Radware module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/snort.asciidoc b/filebeat/docs/modules/snort.asciidoc index 8599c6d3e651..0a5c7c0b1c09 100644 --- a/filebeat/docs/modules/snort.asciidoc +++ b/filebeat/docs/modules/snort.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Snort module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/snort[Snort] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/sonicwall.asciidoc b/filebeat/docs/modules/sonicwall.asciidoc index bd8803659fe3..a760da67ba03 100644 --- a/filebeat/docs/modules/sonicwall.asciidoc +++ b/filebeat/docs/modules/sonicwall.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Sonicwall module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/sonicwall[SonicWall Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/sophos.asciidoc b/filebeat/docs/modules/sophos.asciidoc index 1832e6603c34..a5d12dca366a 100644 --- a/filebeat/docs/modules/sophos.asciidoc +++ b/filebeat/docs/modules/sophos.asciidoc @@ -152,6 +152,8 @@ This is a list of SophosXG fields that are mapped to ECS. [float] ==== `utm` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "astarosg" device revision 123. diff --git a/filebeat/docs/modules/squid.asciidoc b/filebeat/docs/modules/squid.asciidoc index 785fdac83cc8..3f10e40ddd8c 100644 --- a/filebeat/docs/modules/squid.asciidoc +++ b/filebeat/docs/modules/squid.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Squid module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/tomcat.asciidoc b/filebeat/docs/modules/tomcat.asciidoc index bf64a816b052..14aa52b2921a 100644 --- a/filebeat/docs/modules/tomcat.asciidoc +++ b/filebeat/docs/modules/tomcat.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Tomcat module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/apache_tomcat[Apache Tomcat] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/filebeat/docs/modules/zscaler.asciidoc b/filebeat/docs/modules/zscaler.asciidoc index 25f914cb088e..b6adfd84400f 100644 --- a/filebeat/docs/modules/zscaler.asciidoc +++ b/filebeat/docs/modules/zscaler.asciidoc @@ -12,6 +12,8 @@ This file is generated! See scripts/docs_collector.py == Zscaler module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/zscaler_zia[Zscaler Internet Access] Elastic integration instead."] + experimental[] //temporarily override modulename to create working link diff --git a/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc b/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc index 79d27b361660..abca823d9339 100644 --- a/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/barracuda/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Barracuda module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/barracuda[Barracuda Web Application Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc b/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc index 3497b6873ea2..33fe07fd4263 100644 --- a/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/bluecoat/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Bluecoat module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index 6375c1999e8b..eb4777ea0b43 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -274,6 +274,8 @@ include::../include/timezone-support.asciidoc[] [float] ==== `nexus` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_nexus[Cisco Nexus] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "cisconxos" device revision 134. @@ -319,6 +321,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `meraki` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/cisco_meraki[Cisco Meraki] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "ciscomeraki" device revision 118. diff --git a/x-pack/filebeat/module/cylance/_meta/docs.asciidoc b/x-pack/filebeat/module/cylance/_meta/docs.asciidoc index 4cd22f8b7973..f8cff0616f41 100644 --- a/x-pack/filebeat/module/cylance/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cylance/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Cylance module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/f5/_meta/docs.asciidoc b/x-pack/filebeat/module/f5/_meta/docs.asciidoc index e83d14081ff7..28d57c59be58 100644 --- a/x-pack/filebeat/module/f5/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/f5/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == F5 module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/f5_bigip[F5 BIG-IP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc index 2cd30c1437ae..4b224e490191 100644 --- a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc @@ -74,6 +74,8 @@ events. Defaults to `[fortinet-firewall, forwarded]`. [float] ==== `clientendpoint` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_forticlient[Fortinet FortiClient Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "forticlientendpoint" device revision 0. @@ -119,6 +121,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimail` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimail[Fortinet FortiMail] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetfortimail" device revision 131. @@ -164,6 +168,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `fortimanager` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/fortinet_fortimanager[Fortinet FortiManager Logs] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "fortinetmgr" device revision 134. diff --git a/x-pack/filebeat/module/imperva/_meta/docs.asciidoc b/x-pack/filebeat/module/imperva/_meta/docs.asciidoc index c5b1ab376910..6639cabb9b7f 100644 --- a/x-pack/filebeat/module/imperva/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/imperva/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Imperva module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc b/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc index 1be04b5b739f..dc894b8f3c3c 100644 --- a/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/infoblox/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Infoblox module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/infoblox_nios[Infoblox NIOS] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc index 7de15937c4e4..a33b9c03a83c 100644 --- a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc @@ -135,6 +135,8 @@ This is a list of JunOS fields that are mapped to ECS. [float] ==== `junos` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/juniper_srx[Juniper SRX] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "junosrouter" device revision 134. @@ -180,6 +182,8 @@ will be found under `rsa.raw`. The default is false. [float] ==== `netscreen` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. See <> for migration options."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "netscreen" device revision 134. diff --git a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc index b075caa035ba..048354d95a8c 100644 --- a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc @@ -217,6 +217,8 @@ And for all other Defender ATP event types, go to Host -> Events. [float] ==== `dhcp` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/microsoft_dhcp[Microsoft DHCP] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "msdhcp" device revision 99. diff --git a/x-pack/filebeat/module/netscout/_meta/docs.asciidoc b/x-pack/filebeat/module/netscout/_meta/docs.asciidoc index 463c93b5c0f2..6ae6035957ee 100644 --- a/x-pack/filebeat/module/netscout/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/netscout/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Netscout module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] This is a module for receiving Arbor Peakflow SP logs over Syslog or a file. diff --git a/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc b/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc index 21a768ab432d..0e678fa78963 100644 --- a/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/proofpoint/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Proofpoint module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/proofpoint_tap[Proofpoint TAP] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/radware/_meta/docs.asciidoc b/x-pack/filebeat/module/radware/_meta/docs.asciidoc index ba9bc87444c5..834586f8535e 100644 --- a/x-pack/filebeat/module/radware/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/radware/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Radware module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/snort/_meta/docs.asciidoc b/x-pack/filebeat/module/snort/_meta/docs.asciidoc index 3170a20db9d7..025afa52f350 100644 --- a/x-pack/filebeat/module/snort/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/snort/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Snort module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/snort[Snort] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc b/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc index d1f82f17dd24..adccfd17799a 100644 --- a/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/sonicwall/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Sonicwall module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/sonicwall[SonicWall Firewall] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/sophos/_meta/docs.asciidoc b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc index 6d6147984a1d..f998d7ce62cc 100644 --- a/x-pack/filebeat/module/sophos/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc @@ -145,6 +145,8 @@ This is a list of SophosXG fields that are mapped to ECS. [float] ==== `utm` fileset settings +deprecated::[8.12.0,"This fileset is deprecated. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead."] + experimental[] NOTE: This was converted from RSA NetWitness log parser XML "astarosg" device revision 123. diff --git a/x-pack/filebeat/module/squid/_meta/docs.asciidoc b/x-pack/filebeat/module/squid/_meta/docs.asciidoc index 10411c4f73d2..25a8e1edc1ec 100644 --- a/x-pack/filebeat/module/squid/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/squid/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Squid module +deprecated::[8.12.0,"This module is deprecated. See <> for migration options."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc b/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc index d56df0ba01be..cb7339f33881 100644 --- a/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/tomcat/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Tomcat module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/apache_tomcat[Apache Tomcat] Elastic integration instead."] + experimental[] include::{libbeat-dir}/shared/integration-link.asciidoc[] diff --git a/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc b/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc index 1c62e822558e..8a1b07a13724 100644 --- a/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/zscaler/_meta/docs.asciidoc @@ -5,6 +5,8 @@ == Zscaler module +deprecated::[8.12.0,"This module is deprecated. Use the https://docs.elastic.co/integrations/zscaler_zia[Zscaler Internet Access] Elastic integration instead."] + experimental[] //temporarily override modulename to create working link From 2a26f9af28835415d28ba8b439666ce1b9b12bc1 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 26 Oct 2023 14:17:45 -0400 Subject: [PATCH 33/50] go.mod - Use github.com/apache/arrow/go/v12 v12.0.1 (#36953) Move to release tag instead of release candidate. https://github.com/apache/arrow/releases/tag/go%2Fv12.0.1 Closes #36949 --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 0d59be5ef12d..9615dd77f885 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2970,11 +2970,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/apache/arrow/go/v12 -Version: v12.0.1-0.20230605094802-c153c6d36ccf +Version: v12.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/v12@v12.0.1-0.20230605094802-c153c6d36ccf/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/apache/arrow/go/v12@v12.0.1/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 037291b36cd7..9bb1e6803aa1 100644 --- a/go.mod +++ b/go.mod @@ -193,7 +193,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Azure/go-autorest/autorest/adal v0.9.14 - github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf + github.com/apache/arrow/go/v12 v12.0.1 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 diff --git a/go.sum b/go.sum index 8d1a2265e373..038eeca63f6c 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= -github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf h1:s5MDQXJmEalr0Urt0rPlX5UAE2BcHTiex/2Lt2O9p84= -github.com/apache/arrow/go/v12 v12.0.1-0.20230605094802-c153c6d36ccf/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= +github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= +github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= From b0455f428addcb8bc1a3eadec94a6e7f229542cc Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 26 Oct 2023 17:55:17 -0400 Subject: [PATCH 34/50] auditbeat/module - docs and comment nits (#36952) Really minor fixes to docs and comments in auditbeat/modules. As I was reading the code I was applying some changes, so I figured I should contribute them. Removing an unnecessary cast revealed an incorrect operation between `& ^` vs `&^`. It looks like this was supposed to be a bitwise clear `&^` operation to remove ConfigChange and AttributesModified before the lookup from actionOrderMap. --------- Co-authored-by: Dan Kortschak <90160302+efd6@users.noreply.github.com> --- .../docs/modules/file_integrity.asciidoc | 2 +- auditbeat/module/auditd/audit_linux.go | 3 +-- .../module/file_integrity/_meta/docs.asciidoc | 2 +- auditbeat/module/file_integrity/action.go | 2 +- auditbeat/module/file_integrity/config.go | 4 ++-- auditbeat/module/file_integrity/event.go | 8 +++---- .../module/file_integrity/file_parsers.go | 2 +- .../file_integrity/fileorigin_darwin.go | 24 ++++++++++--------- auditbeat/module/file_integrity/metricset.go | 2 +- 9 files changed, 25 insertions(+), 24 deletions(-) diff --git a/auditbeat/docs/modules/file_integrity.asciidoc b/auditbeat/docs/modules/file_integrity.asciidoc index ccc4289c521e..a12c4df47ca0 100644 --- a/auditbeat/docs/modules/file_integrity.asciidoc +++ b/auditbeat/docs/modules/file_integrity.asciidoc @@ -121,7 +121,7 @@ units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, *`max_file_size`*:: The maximum size of a file in bytes for which {beatname_uc} will compute hashes and run file parsers. Files larger than this size will not be hashed or analysed by configured file parsers. The default -value is 100 MiB. For convenience units can be specified as a suffix to the +value is 100 MiB. For convenience, units can be specified as a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index baff3363bed0..9a00b03c4820 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -979,8 +979,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) { if c.SocketType == "" { return "", fmt.Errorf("failed to create audit client: %w", err) } - // Ignore errors if a socket type has been specified. It will fail during - // further setup and its necessary for unit tests to pass + // Ignore errors if a socket type has been specified. return c.SocketType, nil } defer client.Close() diff --git a/auditbeat/module/file_integrity/_meta/docs.asciidoc b/auditbeat/module/file_integrity/_meta/docs.asciidoc index 3c537a280918..0f32ef64f930 100644 --- a/auditbeat/module/file_integrity/_meta/docs.asciidoc +++ b/auditbeat/module/file_integrity/_meta/docs.asciidoc @@ -114,7 +114,7 @@ units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, *`max_file_size`*:: The maximum size of a file in bytes for which {beatname_uc} will compute hashes and run file parsers. Files larger than this size will not be hashed or analysed by configured file parsers. The default -value is 100 MiB. For convenience units can be specified as a suffix to the +value is 100 MiB. For convenience, units can be specified as a suffix to the value. The supported units are `b` (default), `kib`, `kb`, `mib`, `mb`, `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. diff --git a/auditbeat/module/file_integrity/action.go b/auditbeat/module/file_integrity/action.go index 9cfb22747af6..779db7941bd6 100644 --- a/auditbeat/module/file_integrity/action.go +++ b/auditbeat/module/file_integrity/action.go @@ -155,7 +155,7 @@ func (action Action) InOrder(existedBefore, existsNow bool) ActionArray { hasConfigChange := action&ConfigChange != 0 hasUpdate := action&Updated != 0 hasAttrMod := action&AttributesModified != 0 - action = Action(int(action) & int(^(ConfigChange | AttributesModified))) + action = Action(int(action) &^ (ConfigChange | AttributesModified)) if hasAttrMod { action |= Updated } diff --git a/auditbeat/module/file_integrity/config.go b/auditbeat/module/file_integrity/config.go index 1edd19d9ba0e..e431e6407667 100644 --- a/auditbeat/module/file_integrity/config.go +++ b/auditbeat/module/file_integrity/config.go @@ -91,8 +91,8 @@ type Config struct { // Validate validates the config data and return an error explaining all the // problems with the config. This method modifies the given config. func (c *Config) Validate() error { - // Resolve symlinks and make filepaths absolute if possible - // anything that does not resolve will be logged during + // Resolve symlinks and make filepaths absolute if possible. + // Anything that does not resolve will be logged during // scanning and metric set collection. for i, p := range c.Paths { p, err := filepath.EvalSymlinks(p) diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index a86130d3ec87..fd4d68828a44 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -99,7 +99,7 @@ var typeNames = map[Type]string{ SymlinkType: "symlink", } -// Digest is a output of a hash function. +// Digest is an output of a hash function. type Digest []byte // String returns the digest value in lower-case hexadecimal form. @@ -110,7 +110,7 @@ func (d Digest) String() string { // MarshalText encodes the digest to a hexadecimal representation of itself. func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } -// Event describe the filesystem change and includes metadata about the file. +// Event describes the filesystem change and includes metadata about the file. type Event struct { Timestamp time.Time `json:"timestamp"` // Time of event. Path string `json:"path"` // The path associated with the event. @@ -119,7 +119,7 @@ type Event struct { Source Source `json:"source"` // Source of the event. Action Action `json:"action"` // Action (like created, updated). Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. - ParserResults mapstr.M `json:"file,omitempty"` // Results from runnimg file parsers. + ParserResults mapstr.M `json:"file,omitempty"` // Results from running file parsers. // Metadata rtt time.Duration // Time taken to collect the info. @@ -142,7 +142,7 @@ type Metadata struct { Mode os.FileMode `json:"mode"` // Permissions SetUID bool `json:"setuid"` // setuid bit (POSIX only) SetGID bool `json:"setgid"` // setgid bit (POSIX only) - Origin []string `json:"origin"` // External origin info for the file (MacOS only) + Origin []string `json:"origin"` // External origin info for the file (macOS only) SELinux string `json:"selinux"` // security.selinux xattr value (Linux only) POSIXACLAccess []byte `json:"posix_acl_access"` // system.posix_acl_access xattr value (Linux only) } diff --git a/auditbeat/module/file_integrity/file_parsers.go b/auditbeat/module/file_integrity/file_parsers.go index 77f6e30451e8..cb1d619f1ed2 100644 --- a/auditbeat/module/file_integrity/file_parsers.go +++ b/auditbeat/module/file_integrity/file_parsers.go @@ -23,7 +23,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -// FileParser is a file analyser the provides enrichment for file.* fields. +// FileParser is a file analyser providing enrichment for file.* fields. type FileParser interface { Parse(dst mapstr.M, path string) error } diff --git a/auditbeat/module/file_integrity/fileorigin_darwin.go b/auditbeat/module/file_integrity/fileorigin_darwin.go index 1cf6b350d320..459951c8cda6 100644 --- a/auditbeat/module/file_integrity/fileorigin_darwin.go +++ b/auditbeat/module/file_integrity/fileorigin_darwin.go @@ -47,27 +47,29 @@ var ( ) // GetFileOrigin fetches the kMDItemWhereFroms metadata for the given path. This -// is special metadata in the filesystem that encodes information of an external -// origin of this file. It is always encoded as a list of strings, with +// is special metadata in the filesystem that encodes information about the +// external origin of this file. It is always encoded as a list of strings, with // different meanings depending on the origin: // // For files downloaded from a web browser, the first string is the URL for // the source document. The second URL (optional), is the web address where the // download link was followed: -// [ "https://cdn.kernel.org/pub/linux/kernel/v4.x/ChangeLog-4.13.16", "https://www.kernel.org/" ] +// +// ["https://cdn.kernel.org/pub/linux/kernel/v4.x/ChangeLog-4.13.16", "https://www.kernel.org/"] // // For files or directories transferred via Airdrop, the origin is one string // with the name of the computer that sent the file: -// [ "Adrian's MacBook Pro" ] +// +// ["Adrian's MacBook Pro"] // // For files attached to e-mails (using Mail app), three strings are // returned: Sender address, subject and e-mail identifier: -// [ "Adrian Serrano \u003cadrian@elastic.co\u003e", -// -// "Sagrada Familia tickets", -// "message:%3CCAMZw10FD4fktC9qdJgLjwW=a8LM4gbJ44jFcaK8.BOWg1t4OwQ@elastic.co%3E" // -// ], +// [ +// "Adrian Serrano \u003cadrian@elastic.co\u003e", +// "Sagrada Familia tickets", +// "message:%3CCAMZw10FD4fktC9qdJgLjwW=a8LM4gbJ44jFcaK8.BOWg1t4OwQ@elastic.co%3E" +// ], // // For all other files the result is an empty (nil) list. func GetFileOrigin(path string) ([]string, error) { @@ -108,8 +110,8 @@ func GetFileOrigin(path string) ([]string, error) { return nil, fmt.Errorf("plist unmarshal failed: %w", err) } - // The returned list seems to be padded with empty strings when some of - // the fields are missing (i.e. no context URL). Get rid of trailing empty + // The returned list seems to be padded with empty strings when some + // fields are missing (i.e. no context URL). Get rid of trailing empty // strings: n := len(urls) for n > 0 && len(urls[n-1]) == 0 { diff --git a/auditbeat/module/file_integrity/metricset.go b/auditbeat/module/file_integrity/metricset.go index bcada27db9f4..4000231fd335 100644 --- a/auditbeat/module/file_integrity/metricset.go +++ b/auditbeat/module/file_integrity/metricset.go @@ -380,7 +380,7 @@ func store(b datastore.Bucket, e *Event) error { return nil } -// load loads an Event from the datastore. It return a nil Event if the key was +// load loads an Event from the datastore. It returns a nil Event if the key was // not found. It returns an error if there was a failure reading from the // datastore or decoding the data. func load(b datastore.Bucket, path string) (*Event, error) { From 27792cfd516607db718af4975dd790353d9ae559 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Thu, 26 Oct 2023 23:56:52 +0200 Subject: [PATCH 35/50] [Docs] Update packetbeat-options.asciidoc (#36968) Fix packetbeat documentation to avoid '{beatname}' being shown instead of the correct beat name. --- packetbeat/docs/packetbeat-options.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index 4a74dd5593ec..c5cb4d95d6b8 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -57,7 +57,7 @@ packetbeat.interfaces.buffer_size_mb: 100 [float] === Windows Npcap installation options -On Windows {beatname} requires an Npcap DLL installation. This is provided by {beatname} +On Windows {beatname_uc} requires an Npcap DLL installation. This is provided by {beatname_uc} for users of the Elastic Licenced version. In some cases users may wish to use their own installed version. In order to do this the `packetbeat.npcap.never_install` option can be used. Setting this option to `true` will not attempt to install the From ae11b47ef38d5f1b307db27634f5d45aab6f0828 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 27 Oct 2023 13:13:13 -0400 Subject: [PATCH 36/50] Update to elastic-agent-libs v0.6.2. (#36969) * Update to elastic-agent-libs v0.6.2. Increases the version of golang.org/x/net to one that has the HTTP2 rapid reset fix. * Add changelog entry. * Updates for latest version of golang.org/x/net/publicsuffix * Update all remaining subdomains --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 24 ++-- go.mod | 12 +- go.sum | 24 ++-- .../squid/log/test/access1.log-expected.json | 110 +++++++++--------- 5 files changed, 86 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 74a66ef3c23e..37763f34d56a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -174,6 +174,7 @@ is collected by it. - Beats will now connect to older Elasticsearch instances by default {pull}36884[36884] - Raise up logging level to warning when attempting to configure beats with unknown fields from autodiscovered events/environments - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] +- Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 9615dd77f885..ea5514da6ebc 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12712,11 +12712,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.6.0 +Version: v0.6.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.6.2/LICENSE: Apache License Version 2.0, January 2004 @@ -24738,11 +24738,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.12.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24849,11 +24849,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.12.0 +Version: v0.17.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24960,11 +24960,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.12.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24997,11 +24997,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.12.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51107,11 +51107,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.11.0 +Version: v0.13.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index 9bb1e6803aa1..55ef8b4acc5a 100644 --- a/go.mod +++ b/go.mod @@ -152,14 +152,14 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.25.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.14.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.10.0 - golang.org/x/net v0.12.0 + golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.10.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.12.0 - golang.org/x/text v0.12.0 + golang.org/x/sys v0.13.0 + golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.9.1 google.golang.org/api v0.126.0 @@ -202,7 +202,7 @@ require ( github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.6.4 - github.com/elastic/elastic-agent-libs v0.6.0 + github.com/elastic/elastic-agent-libs v0.6.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.7.0 github.com/elastic/go-elasticsearch/v8 v8.10.0 @@ -365,7 +365,7 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.11.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect diff --git a/go.sum b/go.sum index 038eeca63f6c..df5741e581b7 100644 --- a/go.sum +++ b/go.sum @@ -654,8 +654,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.4 h1:K+xC7OGgcy4fLXVuGgOGLs+e github.com/elastic/elastic-agent-autodiscover v0.6.4/go.mod h1:5+7NIBAILc0GkgxYW3ckXncu5wRZfltZhTY4aZAYP4M= github.com/elastic/elastic-agent-client/v7 v7.4.0 h1:h75oTkkvIjgiKVm61NpvTZP4cy6QbQ3zrIpXKGigyjo= github.com/elastic/elastic-agent-client/v7 v7.4.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0= -github.com/elastic/elastic-agent-libs v0.6.0 h1:HnL/OpAzIHlK8y1J69XQuAx4tlCzd6e2kldMHvXARvY= -github.com/elastic/elastic-agent-libs v0.6.0/go.mod h1:K6U+n84siZ66ZyG36h1/x+fw1oIZbFXEypAC6KSiFOg= +github.com/elastic/elastic-agent-libs v0.6.2 h1:tE5pFK4y7xm1FtXm+r+63G7STjJAaWh3+oKIQDzdPDo= +github.com/elastic/elastic-agent-libs v0.6.2/go.mod h1:o+EySawBZGeYu49shJxerg2wRCimS1dhrD4As0MS700= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= github.com/elastic/elastic-agent-system-metrics v0.7.0 h1:qDLY30UDforSd/TfHfqUDiiHSL6Nu6qLXHsKSxz4OuQ= @@ -2028,8 +2028,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2171,8 +2171,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2342,8 +2342,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2352,8 +2352,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2368,8 +2368,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index d7620e206487..f993048245af 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -4825,9 +4825,9 @@ "rsa.time.event_time_str": "1157689377", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 233, "source.ip": "10.105.33.214", @@ -4839,10 +4839,10 @@ "url.extension": "js", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", "url.path": "/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -4887,9 +4887,9 @@ "rsa.time.event_time_str": "1157689377", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 236, "source.ip": "10.105.33.214", @@ -4901,10 +4901,10 @@ "url.extension": "css", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", "url.path": "/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -4949,9 +4949,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -4963,10 +4963,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5068,9 +5068,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 136, "source.ip": "10.105.33.214", @@ -5082,10 +5082,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", "url.path": "/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5130,9 +5130,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 237, "source.ip": "10.105.33.214", @@ -5144,10 +5144,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5190,9 +5190,9 @@ "rsa.time.event_time_str": "1157689378", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 218, "source.ip": "10.105.33.214", @@ -5204,10 +5204,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", "url.path": "/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5252,9 +5252,9 @@ "rsa.time.event_time_str": "1157689379", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5266,10 +5266,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5314,9 +5314,9 @@ "rsa.time.event_time_str": "1157689379", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5328,10 +5328,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", "url.path": "/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -5376,9 +5376,9 @@ "rsa.time.event_time_str": "1157689380", "rsa.web.alias_host": "a1568.g.akamai.net", "server.domain": "a1568.g.akamai.net", - "server.registered_domain": "akamai.net", - "server.subdomain": "a1568.g", - "server.top_level_domain": "net", + "server.registered_domain": "g.akamai.net", + "server.subdomain": "a1568", + "server.top_level_domain": "akamai.net", "service.type": "squid", "source.bytes": 238, "source.ip": "10.105.33.214", @@ -5390,10 +5390,10 @@ "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", - "url.registered_domain": "akamai.net", + "url.registered_domain": "g.akamai.net", "url.scheme": "http", - "url.subdomain": "a1568.g", - "url.top_level_domain": "net", + "url.subdomain": "a1568", + "url.top_level_domain": "akamai.net", "user.name": "adeolaegbedokun" }, { @@ -6059,4 +6059,4 @@ "url.top_level_domain": "com", "user.name": "badeyek" } -] \ No newline at end of file +] From a0669d2d9c96ee28709a88fe29a502e325675c90 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Fri, 27 Oct 2023 11:26:42 -0700 Subject: [PATCH 37/50] Fix pipeline setup under serverless (#36973) * fix pipeline setup under serverless * add docs, fix errors * fix fix linter --- filebeat/fileset/pipelines.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/filebeat/fileset/pipelines.go b/filebeat/fileset/pipelines.go index 07d6c4c21ac3..c58a7c13edcf 100644 --- a/filebeat/fileset/pipelines.go +++ b/filebeat/fileset/pipelines.go @@ -37,6 +37,7 @@ type PipelineLoader interface { LoadJSON(path string, json map[string]interface{}) ([]byte, error) Request(method, path string, pipeline string, params map[string]string, body interface{}) (int, []byte, error) GetVersion() version.V + IsServerless() bool } // MultiplePipelineUnsupportedError is an error returned when a fileset uses multiple pipelines but is @@ -65,16 +66,17 @@ func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader, overwrite bool // check that all the required Ingest Node plugins are available requiredProcessors := fileset.GetRequiredProcessors() reg.log.Debugf("Required processors: %s", requiredProcessors) - if len(requiredProcessors) > 0 { + // APIs do not exist on serverless + if len(requiredProcessors) > 0 && !esClient.IsServerless() { err := checkAvailableProcessors(esClient, requiredProcessors) if err != nil { - return fmt.Errorf("error loading pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + return fmt.Errorf("error loading pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) } } pipelines, err := fileset.GetPipelines(esClient.GetVersion()) if err != nil { - return fmt.Errorf("error getting pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + return fmt.Errorf("error getting pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) } // Filesets with multiple pipelines can only be supported by Elasticsearch >= 6.5.0 @@ -88,7 +90,7 @@ func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader, overwrite bool for _, pipeline := range pipelines { err = LoadPipeline(esClient, pipeline.id, pipeline.contents, overwrite, reg.log.With("pipeline", pipeline.id)) if err != nil { - err = fmt.Errorf("error loading pipeline for fileset %s/%s: %v", module.config.Module, fileset.name, err) + err = fmt.Errorf("error loading pipeline for fileset %s/%s: %w", module.config.Module, fileset.name, err) break } pipelineIDsLoaded = append(pipelineIDsLoaded, pipeline.id) @@ -169,7 +171,7 @@ func interpretError(initialErr error, body []byte) error { "This is the response I got from Elasticsearch: %s", body) } - return fmt.Errorf("couldn't load pipeline: %v. Additionally, error decoding response body: %s", + return fmt.Errorf("couldn't load pipeline: %w. Additionally, error decoding response body: %s", initialErr, body) } @@ -194,5 +196,5 @@ func interpretError(initialErr error, body []byte) error { "This is the response I got from Elasticsearch: %s", body) } - return fmt.Errorf("couldn't load pipeline: %v. Response body: %s", initialErr, body) + return fmt.Errorf("couldn't load pipeline: %w. Response body: %s", initialErr, body) } From fe5088338011c48230e7d6aa6ce07fed5a895e5c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 27 Oct 2023 14:49:53 -0400 Subject: [PATCH 38/50] Auditbeat,Metricbeat - add /inputs/ to HTTP monitoring endpoint (#36971) Make metrics published by "inputs" available through the /inputs/ route on the HTTP monitoring endpoint of Auditbeat and Metricbeat. For Agent, include a snapshot of those metrics within the Agent diagnostics bundle as "input_metrics.json". When running under Agent, each module instance is configured with only a single metricset. That module is given a unique `id`. That ID is what will be used as the `id` within the /inputs/ data. And that `id` will also be added as context to the logger that is passed into every metricset so that any log messages from a metricset can be associated back to the agent stream ID). Relates #36945 Remove module and metricset keys from metricset metrics. For the `/inputs/` API, `input` is they key used to identify the type of "input" running. The `module` and `metricset` keys become redundant with the addition of `input`. I don't know of anything that relies on those fields. --- CHANGELOG.next.asciidoc | 2 ++ metricbeat/beater/metricbeat.go | 19 +++++++++++++++++++ metricbeat/mb/builders.go | 21 +++++++++++++++++---- metricbeat/mb/mb.go | 5 +++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 37763f34d56a..baf3dd3e7ec0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -253,6 +253,7 @@ is collected by it. *Auditbeat* - Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964] +- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each dataset instance. {pull}36971[36971] *Libbeat* @@ -268,6 +269,7 @@ is collected by it. - Add GCP Carbon Footprint metricbeat data {pull}34820[34820] - Add event loop utilization metric to Kibana module {pull}35020[35020] - Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms {pull}36647[36647] +- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] *Osquerybeat* diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index cbc44f88bf57..acd4aa02b1e3 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/module" conf "github.com/elastic/elastic-agent-libs/config" @@ -155,6 +156,24 @@ func newMetricbeat(b *beat.Beat, c *conf.C, options ...Option) (*Metricbeat, err return metricbeat, nil } + if b.API != nil { + if err := inputmon.AttachHandler(b.API.Router()); err != nil { + return nil, fmt.Errorf("failed attach inputs api to monitoring endpoint server: %w", err) + } + } + + if b.Manager != nil { + b.Manager.RegisterDiagnosticHook("input_metrics", "Metrics from active inputs.", + "input_metrics.json", "application/json", func() []byte { + data, err := inputmon.MetricSnapshotJSON() + if err != nil { + logp.L().Warnw("Failed to collect input metric snapshot for Agent diagnostics.", "error", err) + return []byte(err.Error()) + } + return data + }) + } + moduleOptions := append( []module.Option{module.WithMaxStartDelay(config.MaxStartDelay)}, metricbeat.moduleOptions...) diff --git a/metricbeat/mb/builders.go b/metricbeat/mb/builders.go index 269c194063cb..c9b1ace587d9 100644 --- a/metricbeat/mb/builders.go +++ b/metricbeat/mb/builders.go @@ -182,20 +182,33 @@ func newBaseMetricSets(r *Register, m Module) ([]BaseMetricSet, error) { } msID := id.String() metrics := monitoring.NewRegistry() - monitoring.NewString(metrics, "module").Set(m.Name()) - monitoring.NewString(metrics, "metricset").Set(name) + monitoring.NewString(metrics, "input").Set(m.Name() + "/" + name) if host != "" { monitoring.NewString(metrics, "host").Set(host) } - monitoring.NewString(metrics, "id").Set(msID) + monitoring.NewString(metrics, "ephemeral_id").Set(msID) + if configuredID := m.Config().ID; configuredID != "" { + // If a module ID was configured, then use that as the ID within metrics. + // Note that the "ephemeral_id" is what is used as the monitoring registry + // key. This module ID is not unique to the MetricSet instance when multiple + // hosts are monitored or if multiple different MetricSet types were enabled + // under the same module instance. + monitoring.NewString(metrics, "id").Set(configuredID) + } else { + monitoring.NewString(metrics, "id").Set(msID) + } + logger := logp.NewLogger(m.Name() + "." + name) + if m.Config().ID != "" { + logger = logger.With("id", m.Config().ID) + } metricsets = append(metricsets, BaseMetricSet{ id: msID, name: name, module: m, host: host, metrics: metrics, - logger: logp.NewLogger(m.Name() + "." + name), + logger: logger, }) } } diff --git a/metricbeat/mb/mb.go b/metricbeat/mb/mb.go index 06b85662838c..7e18dc9029d2 100644 --- a/metricbeat/mb/mb.go +++ b/metricbeat/mb/mb.go @@ -362,6 +362,7 @@ func (b *BaseMetricSet) Registration() MetricSetRegistration { // the metricset fetches not only the predefined fields but add alls raw data under // the raw namespace to the event. type ModuleConfig struct { + ID string `config:"id"` // Optional ID (not guaranteed to be unique). Hosts []string `config:"hosts"` Period time.Duration `config:"period" validate:"positive"` Timeout time.Duration `config:"timeout" validate:"positive"` @@ -375,8 +376,8 @@ type ModuleConfig struct { func (c ModuleConfig) String() string { return fmt.Sprintf(`{Module:"%v", MetricSets:%v, Enabled:%v, `+ - `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, - c.Module, c.MetricSets, c.Enabled, len(c.Hosts), c.Period, c.Timeout, + `ID:"%s", Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, + c.Module, c.MetricSets, c.Enabled, c.ID, len(c.Hosts), c.Period, c.Timeout, c.Raw, c.Query) } From 4453d992cc659ef0655cf6cc19488da66135b193 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Sat, 28 Oct 2023 06:50:26 +1030 Subject: [PATCH 39/50] auditbeat/module/auditd: add ignore_errors config option (#36851) Setting ignore_errors to true allows incompletely valid rule sets to be used in a configuration. This is equivalent to the -i flag of auditctl. --- CHANGELOG.next.asciidoc | 1 + auditbeat/docs/modules/auditd.asciidoc | 3 + auditbeat/module/auditd/_meta/docs.asciidoc | 3 + .../auditd/{config_linux.go => config.go} | 37 ++- auditbeat/module/auditd/config_linux_test.go | 254 ---------------- auditbeat/module/auditd/config_test.go | 279 ++++++++++++++++++ 6 files changed, 316 insertions(+), 261 deletions(-) rename auditbeat/module/auditd/{config_linux.go => config.go} (83%) delete mode 100644 auditbeat/module/auditd/config_linux_test.go create mode 100644 auditbeat/module/auditd/config_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index baf3dd3e7ec0..e833f99d0b11 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -178,6 +178,7 @@ is collected by it. *Auditbeat* +- Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851] *Filebeat* diff --git a/auditbeat/docs/modules/auditd.asciidoc b/auditbeat/docs/modules/auditd.asciidoc index a0d2693487e3..9204e243f644 100644 --- a/auditbeat/docs/modules/auditd.asciidoc +++ b/auditbeat/docs/modules/auditd.asciidoc @@ -212,6 +212,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are supported and will expand in lexicographical order. The format is the same as that of the `audit_rules` field. +*`ignore_errors`*:: This setting allows errors during rule loading and parsing +to be ignored, but logged as warnings. + *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to prevent backpressure from propagating to the kernel and impacting audited processes. diff --git a/auditbeat/module/auditd/_meta/docs.asciidoc b/auditbeat/module/auditd/_meta/docs.asciidoc index 587a40dd9824..b1dd7d87c633 100644 --- a/auditbeat/module/auditd/_meta/docs.asciidoc +++ b/auditbeat/module/auditd/_meta/docs.asciidoc @@ -205,6 +205,9 @@ loaded after the rules declared in `audit_rules` are loaded. Wildcards are supported and will expand in lexicographical order. The format is the same as that of the `audit_rules` field. +*`ignore_errors`*:: This setting allows errors during rule loading and parsing +to be ignored, but logged as warnings. + *`backpressure_strategy`*:: Specifies the strategy that {beatname_uc} uses to prevent backpressure from propagating to the kernel and impacting audited processes. diff --git a/auditbeat/module/auditd/config_linux.go b/auditbeat/module/auditd/config.go similarity index 83% rename from auditbeat/module/auditd/config_linux.go rename to auditbeat/module/auditd/config.go index 4a03584c3b7b..6762a3924a9d 100644 --- a/auditbeat/module/auditd/config_linux.go +++ b/auditbeat/module/auditd/config.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//go:build unix + package auditd import ( @@ -30,6 +32,7 @@ import ( "github.com/joeshaw/multierror" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-libaudit/v2/rule" "github.com/elastic/go-libaudit/v2/rule/flags" ) @@ -46,6 +49,7 @@ type Config struct { RuleFiles []string `config:"audit_rule_files"` // List of rule files. SocketType string `config:"socket_type"` // Socket type to use with the kernel (unicast or multicast). Immutable bool `config:"immutable"` // Sets kernel audit config immutable. + IgnoreErrors bool `config:"ignore_errors"` // Ignore errors when reading and parsing rules, equivalent to auditctl -i. // Tuning options (advanced, use with care) ReassemblerMaxInFlight uint32 `config:"reassembler.max_in_flight"` @@ -120,11 +124,19 @@ func (c Config) rules() []auditRule { } func (c *Config) loadRules() error { + var log *logp.Logger + if c.IgnoreErrors { + log = logp.NewLogger(moduleName) + } + var paths []string for _, pattern := range c.RuleFiles { absPattern, err := filepath.Abs(pattern) if err != nil { - return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err) + if log == nil { + return fmt.Errorf("unable to get the absolute path for %s: %w", pattern, err) + } + log.Warnf("unable to get the absolute path for %s: %v", pattern, err) } files, err := filepath.Glob(absPattern) if err != nil { @@ -136,7 +148,7 @@ func (c *Config) loadRules() error { knownRules := ruleSet{} - rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules) + rules, err := readRules(bytes.NewBufferString(c.RulesBlob), "(audit_rules at auditbeat.yml)", knownRules, log) if err != nil { return err } @@ -145,9 +157,13 @@ func (c *Config) loadRules() error { for _, filename := range paths { fHandle, err := os.Open(filename) if err != nil { - return fmt.Errorf("unable to open rule file '%s': %w", filename, err) + if log == nil { + return fmt.Errorf("unable to open rule file '%s': %w", filename, err) + } + log.Warnf("unable to open rule file '%s': %v", filename, err) + continue } - rules, err = readRules(fHandle, filename, knownRules) + rules, err = readRules(fHandle, filename, knownRules, log) if err != nil { return err } @@ -170,7 +186,11 @@ func (c Config) failureMode() (uint32, error) { } } -func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []auditRule, err error) { +// readRules reads the audit rules from reader, adding them to knownRules. If +// log is nil, errors will result in an empty rules set being returned. Otherwise +// errors will be logged as warnings and any successfully parsed rules will be +// returned. +func readRules(reader io.Reader, source string, knownRules ruleSet, log *logp.Logger) (rules []auditRule, err error) { var errs multierror.Errors s := bufio.NewScanner(reader) @@ -207,8 +227,11 @@ func readRules(reader io.Reader, source string, knownRules ruleSet) (rules []aud rules = append(rules, rule) } - if len(errs) > 0 { - return nil, fmt.Errorf("failed loading rules: %w", errs.Err()) + if len(errs) != 0 { + if log == nil { + return nil, fmt.Errorf("failed loading rules: %w", errs.Err()) + } + log.Warnf("errors loading rules: %v", errs.Err()) } return rules, nil } diff --git a/auditbeat/module/auditd/config_linux_test.go b/auditbeat/module/auditd/config_linux_test.go deleted file mode 100644 index d48eea8c6c8d..000000000000 --- a/auditbeat/module/auditd/config_linux_test.go +++ /dev/null @@ -1,254 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package auditd - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - - conf "github.com/elastic/elastic-agent-libs/config" -) - -func TestConfigValidate(t *testing.T) { - data := ` -audit_rules: | - # Comments and empty lines are ignored. - -w /etc/passwd -p wa -k auth - - -a always,exit -S execve -k exec` - - config, err := parseConfig(t, data) - if err != nil { - t.Fatal(err) - } - rules := config.rules() - - assert.EqualValues(t, []string{ - "-w /etc/passwd -p wa -k auth", - "-a always,exit -S execve -k exec", - }, commands(rules)) -} - -func TestConfigValidateWithError(t *testing.T) { - data := ` -audit_rules: | - -x bad -F flag - -a always,exit -w /etc/passwd - -a always,exit -S fake -k exec` - - _, err := parseConfig(t, data) - if err == nil { - t.Fatal("expected error") - } - t.Log(err) -} - -func TestConfigValidateWithDuplicates(t *testing.T) { - data := ` -audit_rules: | - -w /etc/passwd -p rwxa -k auth - -w /etc/passwd -k auth` - - _, err := parseConfig(t, data) - if err == nil { - t.Fatal("expected error") - } - t.Log(err) -} - -func TestConfigValidateFailureMode(t *testing.T) { - config := defaultConfig - config.FailureMode = "boom" - err := config.Validate() - assert.Error(t, err) - t.Log(err) -} - -func TestConfigValidateConnectionType(t *testing.T) { - config := defaultConfig - config.SocketType = "Satellite" - err := config.Validate() - assert.Error(t, err) - t.Log(err) -} - -func TestConfigValidateImmutable(t *testing.T) { - tcs := []struct { - name string - socketType string - mustFail bool - }{ - { - name: "Must pass for default", - socketType: "", - mustFail: false, - }, - { - name: "Must pass for unicast", - socketType: "unicast", - mustFail: false, - }, - { - name: "Must fail for multicast", - socketType: "multicast", - mustFail: true, - }, - } - - for _, tc := range tcs { - tc := tc - t.Run(tc.name, func(t *testing.T) { - config := defaultConfig - config.SocketType = tc.socketType - config.Immutable = true - err := config.Validate() - if tc.mustFail { - assert.Error(t, err) - t.Log(err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestConfigRuleOrdering(t *testing.T) { - const fileMode = 0o644 - config := defaultConfig - config.RulesBlob = strings.Join([]string{ - makeRuleFlags(0, 0), - makeRuleFlags(0, 1), - makeRuleFlags(0, 2), - }, "\n") - - dir1, err := ioutil.TempDir("", "rules1") - if err != nil { - t.Fatal(err) - } - - for _, file := range []struct { - order int - name string - }{ - {0, "00_first.conf"}, - {5, "99_last.conf"}, - {2, "03_auth.conf"}, - {4, "20_exec.conf"}, - {3, "10_network_access.conf"}, - {1, "01_32bit_abi.conf"}, - } { - path := filepath.Join(dir1, file.name) - content := []byte(strings.Join([]string{ - makeRuleFlags(1+file.order, 0), - makeRuleFlags(1+file.order, 1), - makeRuleFlags(1+file.order, 2), - makeRuleFlags(1+file.order, 3), - }, "\n")) - if err = ioutil.WriteFile(path, content, fileMode); err != nil { - t.Fatal(err) - } - } - - dir2, err := ioutil.TempDir("", "rules0") - if err != nil { - t.Fatal(err) - } - - for _, file := range []struct { - order int - name string - }{ - {3, "99_tail.conf"}, - {0, "00_head.conf"}, - {2, "50_mid.conf"}, - {1, "13.conf"}, - } { - path := filepath.Join(dir2, file.name) - content := []byte(strings.Join([]string{ - makeRuleFlags(10+file.order, 0), - makeRuleFlags(10+file.order, 1), - makeRuleFlags(10+file.order, 2), - makeRuleFlags(10+file.order, 3), - }, "\n")) - if err = ioutil.WriteFile(path, content, fileMode); err != nil { - t.Fatal(err) - } - } - - config.RuleFiles = []string{ - fmt.Sprintf("%s/*.conf", dir1), - fmt.Sprintf("%s/*.conf", dir2), - } - - if err = config.Validate(); err != nil { - t.Fatal(err) - } - - rules := config.rules() - fileNo, ruleNo := 0, 0 - for _, rule := range rules { - parts := strings.Split(rule.flags, " ") - assert.Len(t, parts, 6, rule.flags) - fields := strings.Split(parts[5], ":") - assert.Len(t, fields, 3, rule.flags) - fileID, err := strconv.Atoi(fields[1]) - if err != nil { - t.Fatal(err, rule.flags) - } - ruleID, err := strconv.Atoi(fields[2]) - if err != nil { - t.Fatal(err, rule.flags) - } - if fileID > fileNo { - fileNo = fileID - ruleNo = 0 - } - assert.Equal(t, fileNo, fileID, rule.flags) - assert.Equal(t, ruleNo, ruleID, rule.flags) - ruleNo++ - } -} - -func makeRuleFlags(fileID, ruleID int) string { - return fmt.Sprintf("-w /path/%d/%d -p rwxa -k rule:%d:%d", fileID, ruleID, fileID, ruleID) -} - -func parseConfig(t testing.TB, yaml string) (Config, error) { - c, err := conf.NewConfigWithYAML([]byte(yaml), "") - if err != nil { - t.Fatal(err) - } - - config := defaultConfig - err = c.Unpack(&config) - return config, err -} - -func commands(rules []auditRule) []string { - var cmds []string - for _, r := range rules { - cmds = append(cmds, r.flags) - } - return cmds -} diff --git a/auditbeat/module/auditd/config_test.go b/auditbeat/module/auditd/config_test.go new file mode 100644 index 000000000000..81da2d9b85ba --- /dev/null +++ b/auditbeat/module/auditd/config_test.go @@ -0,0 +1,279 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build unix + +package auditd + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestConfig(t *testing.T) { + logp.TestingSetup() + + t.Run("Validate", func(t *testing.T) { + data := ` +audit_rules: | + # Comments and empty lines are ignored. + -w /etc/passwd -p wa -k auth + + -a always,exit -S execve -k exec` + + config, err := parseConfig(t, data) + if err != nil { + t.Fatal(err) + } + rules := config.rules() + + assert.EqualValues(t, []string{ + "-w /etc/passwd -p wa -k auth", + "-a always,exit -S execve -k exec", + }, commands(rules)) + }) + + t.Run("ValidateWithError", func(t *testing.T) { + data := ` +audit_rules: | + -x bad -F flag + -a always,exit -w /etc/passwd + -a always,exit -S fake -k exec` + + _, err := parseConfig(t, data) + if err == nil { + t.Fatal("expected error") + } + t.Log(err) + }) + + t.Run("ValidateWithErrorIgnored", func(t *testing.T) { + data := ` +ignore_errors: true +audit_rules: | + -x bad -F flag + -a always,exit -w /etc/passwd + -a always,exit -S fake -k exec + -w /etc/passwd -k auth` + + cfg, err := parseConfig(t, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cfg.auditRules) != 1 { + t.Fatalf("unexpected number of rules from parseConfig: got %d, want %d", len(cfg.auditRules), 1) + } + }) + + t.Run("ValidateWithDuplicates", func(t *testing.T) { + data := ` +audit_rules: | + -w /etc/passwd -p rwxa -k auth + -w /etc/passwd -k auth` + + _, err := parseConfig(t, data) + if err == nil { + t.Fatal("expected error") + } + t.Log(err) + }) + + t.Run("ValidateFailureMode", func(t *testing.T) { + config := defaultConfig + config.FailureMode = "boom" + err := config.Validate() + assert.Error(t, err) + t.Log(err) + }) + + t.Run("ValidateConnectionType", func(t *testing.T) { + config := defaultConfig + config.SocketType = "Satellite" + err := config.Validate() + assert.Error(t, err) + t.Log(err) + }) + + t.Run("ValidateImmutable", func(t *testing.T) { + tcs := []struct { + name string + socketType string + mustFail bool + }{ + { + name: "Must pass for default", + socketType: "", + mustFail: false, + }, + { + name: "Must pass for unicast", + socketType: "unicast", + mustFail: false, + }, + { + name: "Must fail for multicast", + socketType: "multicast", + mustFail: true, + }, + } + + for _, tc := range tcs { + tc := tc + t.Run(tc.name, func(t *testing.T) { + config := defaultConfig + config.SocketType = tc.socketType + config.Immutable = true + err := config.Validate() + if tc.mustFail { + assert.Error(t, err) + t.Log(err) + } else { + assert.NoError(t, err) + } + }) + } + }) + + t.Run("RuleOrdering", func(t *testing.T) { + const fileMode = 0o644 + config := defaultConfig + config.RulesBlob = strings.Join([]string{ + makeRuleFlags(0, 0), + makeRuleFlags(0, 1), + makeRuleFlags(0, 2), + }, "\n") + + dir1, err := ioutil.TempDir("", "rules1") + if err != nil { + t.Fatal(err) + } + + for _, file := range []struct { + order int + name string + }{ + {0, "00_first.conf"}, + {5, "99_last.conf"}, + {2, "03_auth.conf"}, + {4, "20_exec.conf"}, + {3, "10_network_access.conf"}, + {1, "01_32bit_abi.conf"}, + } { + path := filepath.Join(dir1, file.name) + content := []byte(strings.Join([]string{ + makeRuleFlags(1+file.order, 0), + makeRuleFlags(1+file.order, 1), + makeRuleFlags(1+file.order, 2), + makeRuleFlags(1+file.order, 3), + }, "\n")) + if err = ioutil.WriteFile(path, content, fileMode); err != nil { + t.Fatal(err) + } + } + + dir2, err := ioutil.TempDir("", "rules0") + if err != nil { + t.Fatal(err) + } + + for _, file := range []struct { + order int + name string + }{ + {3, "99_tail.conf"}, + {0, "00_head.conf"}, + {2, "50_mid.conf"}, + {1, "13.conf"}, + } { + path := filepath.Join(dir2, file.name) + content := []byte(strings.Join([]string{ + makeRuleFlags(10+file.order, 0), + makeRuleFlags(10+file.order, 1), + makeRuleFlags(10+file.order, 2), + makeRuleFlags(10+file.order, 3), + }, "\n")) + if err = ioutil.WriteFile(path, content, fileMode); err != nil { + t.Fatal(err) + } + } + + config.RuleFiles = []string{ + fmt.Sprintf("%s/*.conf", dir1), + fmt.Sprintf("%s/*.conf", dir2), + } + + if err = config.Validate(); err != nil { + t.Fatal(err) + } + + rules := config.rules() + fileNo, ruleNo := 0, 0 + for _, rule := range rules { + parts := strings.Split(rule.flags, " ") + assert.Len(t, parts, 6, rule.flags) + fields := strings.Split(parts[5], ":") + assert.Len(t, fields, 3, rule.flags) + fileID, err := strconv.Atoi(fields[1]) + if err != nil { + t.Fatal(err, rule.flags) + } + ruleID, err := strconv.Atoi(fields[2]) + if err != nil { + t.Fatal(err, rule.flags) + } + if fileID > fileNo { + fileNo = fileID + ruleNo = 0 + } + assert.Equal(t, fileNo, fileID, rule.flags) + assert.Equal(t, ruleNo, ruleID, rule.flags) + ruleNo++ + } + }) +} + +func makeRuleFlags(fileID, ruleID int) string { + return fmt.Sprintf("-w /path/%d/%d -p rwxa -k rule:%d:%d", fileID, ruleID, fileID, ruleID) +} + +func parseConfig(t testing.TB, yaml string) (Config, error) { + c, err := conf.NewConfigWithYAML([]byte(yaml), "") + if err != nil { + t.Fatal(err) + } + + config := defaultConfig + err = c.Unpack(&config) + return config, err +} + +func commands(rules []auditRule) []string { + var cmds []string + for _, r := range rules { + cmds = append(cmds, r.flags) + } + return cmds +} From 9dd92030bd796db3724828f318cff0ff15249279 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:15:48 -0400 Subject: [PATCH 40/50] [updatecli] update elastic stack version for testing 8.12.0-fa958a90 (#36864) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3d7bbd0bacb3..28bb4bb0c7f8 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-fa958a90-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-fa958a90-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-cb808527-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-fa958a90-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From adcd4b0c7cc2ef43b4b95ff5ed73d2c9f5a58548 Mon Sep 17 00:00:00 2001 From: Richa Talwar <102972658+ritalwar@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:03:20 +0530 Subject: [PATCH 41/50] Fix event mapping implementation for statsd module (#36925) * Fix eventmapping implementation for statsd module. --- CHANGELOG.next.asciidoc | 1 + .../module/airflow/statsd/_meta/data.json | 8 ++-- .../module/airflow/statsd/data_test.go | 16 +++---- .../metricbeat/module/statsd/server/data.go | 20 +++++---- .../module/statsd/server/data_test.go | 22 +++++++--- .../metricbeat/module/statsd/server/server.go | 42 +++++++++++-------- 6 files changed, 64 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e833f99d0b11..eef14a45f562 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -143,6 +143,7 @@ is collected by it. - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413] - Add log error when statsd server fails to start {pull}36477[36477] - Fix CassandraConnectionClosures metric configuration {pull}34742[34742] +- Fix event mapping implementation for statsd module {pull}36925[36925] *Osquerybeat* diff --git a/x-pack/metricbeat/module/airflow/statsd/_meta/data.json b/x-pack/metricbeat/module/airflow/statsd/_meta/data.json index 4e5c41437cb9..ff89b0ddde23 100644 --- a/x-pack/metricbeat/module/airflow/statsd/_meta/data.json +++ b/x-pack/metricbeat/module/airflow/statsd/_meta/data.json @@ -2,13 +2,13 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "airflow": { "dag_duration": { - "15m_rate": 0.2, - "1m_rate": 0.2, - "5m_rate": 0.2, + "15m_rate": 0, + "1m_rate": 0, + "5m_rate": 0, "count": 1, "max": 200, "mean": 200, - "mean_rate": 0.2222490946071946, + "mean_rate": 38960.532980091164, "median": 200, "min": 200, "p75": 200, diff --git a/x-pack/metricbeat/module/airflow/statsd/data_test.go b/x-pack/metricbeat/module/airflow/statsd/data_test.go index c2c07d32f34b..fcfd943bc686 100644 --- a/x-pack/metricbeat/module/airflow/statsd/data_test.go +++ b/x-pack/metricbeat/module/airflow/statsd/data_test.go @@ -11,15 +11,14 @@ import ( "sync" "testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/auditbeat/core" _ "github.com/elastic/beats/v7/libbeat/processors/actions" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" ) func init() { @@ -42,14 +41,14 @@ func getConfig() map[string]interface{} { } } -func createEvent(t *testing.T) { +func createEvent(data string, t *testing.T) { udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", STATSD_HOST, STATSD_PORT)) require.NoError(t, err) conn, err := net.DialUDP("udp", nil, udpAddr) require.NoError(t, err) - _, err = fmt.Fprint(conn, "dagrun.duration.failed.a_dagid:200|ms|#k1:v1,k2:v2") + _, err = fmt.Fprint(conn, data) require.NoError(t, err) } @@ -70,15 +69,16 @@ func TestData(t *testing.T) { wg.Done() go ms.Run(reporter) - events = reporter.(*mbtest.CapturingPushReporterV2).BlockingCapture(1) + events = reporter.(*mbtest.CapturingPushReporterV2).BlockingCapture(2) close(done) }(wg) wg.Wait() - createEvent(t) + createEvent("dagrun.duration.failed.a_dagid:200|ms|#k1:v1,k2:v2", t) + createEvent("dagrun.duration.failed.b_dagid:500|ms|#k1:v1,k2:v2", t) <-done - + assert.Len(t, events, 2) if len(events) == 0 { t.Fatal("received no events") } diff --git a/x-pack/metricbeat/module/statsd/server/data.go b/x-pack/metricbeat/module/statsd/server/data.go index bac70457fd20..27024e262846 100644 --- a/x-pack/metricbeat/module/statsd/server/data.go +++ b/x-pack/metricbeat/module/statsd/server/data.go @@ -102,18 +102,18 @@ func parse(b []byte) ([]statsdMetric, error) { return metrics, nil } -func eventMapping(metricName string, metricValue interface{}, metricSetFields mapstr.M, mappings map[string]StatsdMapping) { +func eventMapping(metricName string, metricValue interface{}, mappings map[string]StatsdMapping) mapstr.M { + m := mapstr.M{} if len(mappings) == 0 { - metricSetFields[common.DeDot(metricName)] = metricValue - return + m[common.DeDot(metricName)] = metricValue + return m } for _, mapping := range mappings { // The metricname match the one with no labels in mappings - // Let's insert it dedotted and continue if metricName == mapping.Metric { - metricSetFields[mapping.Value.Field] = metricValue - return + m[mapping.Value.Field] = metricValue + return m } res := mapping.regex.FindStringSubmatch(metricName) @@ -121,7 +121,7 @@ func eventMapping(metricName string, metricValue interface{}, metricSetFields ma // Not all labels match // Skip and continue to next mapping if len(res) != (len(mapping.Labels) + 1) { - logger.Debug("not all labels match in statsd.mapping, skipped") + logger.Debug("not all labels match in statsd.mappings, skipped") continue } @@ -133,13 +133,15 @@ func eventMapping(metricName string, metricValue interface{}, metricSetFields ma continue } - metricSetFields[label.Field] = res[i] + m[label.Field] = res[i] } } // Let's add the metric with the value field - metricSetFields[mapping.Value.Field] = metricValue + m[mapping.Value.Field] = metricValue + break } + return m } func newMetricProcessor(ttl time.Duration) *metricProcessor { diff --git a/x-pack/metricbeat/module/statsd/server/data_test.go b/x-pack/metricbeat/module/statsd/server/data_test.go index 2377a2fae5b2..2bdc97ab5c29 100644 --- a/x-pack/metricbeat/module/statsd/server/data_test.go +++ b/x-pack/metricbeat/module/statsd/server/data_test.go @@ -737,11 +737,9 @@ func TestEventMapping(t *testing.T) { }, } { t.Run(test.metricName, func(t *testing.T) { - metricSetFields := mapstr.M{} builtMappings, _ := buildMappings(mappings) - eventMapping(test.metricName, test.metricValue, metricSetFields, builtMappings) - - assert.Equal(t, test.expected, metricSetFields) + ms := eventMapping(test.metricName, test.metricValue, builtMappings) + assert.Equal(t, test.expected, ms) }) } } @@ -1132,7 +1130,7 @@ func TestTagsGrouping(t *testing.T) { require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 2) + assert.Len(t, events, 4) actualTags := []mapstr.M{} for _, e := range events { @@ -1146,6 +1144,18 @@ func TestTagsGrouping(t *testing.T) { "k2": "v2", }, }, + { + "labels": mapstr.M{ + "k1": "v1", + "k2": "v2", + }, + }, + { + "labels": mapstr.M{ + "k1": "v2", + "k2": "v3", + }, + }, { "labels": mapstr.M{ "k1": "v2", @@ -1224,7 +1234,7 @@ func TestData(t *testing.T) { require.NoError(t, err) events := ms.getEvents() - assert.Len(t, events, 1) + assert.Len(t, events, 10) mbevent := mbtest.StandardizeEvent(ms, *events[0]) mbtest.WriteEventToDataJSON(t, mbevent, "") diff --git a/x-pack/metricbeat/module/statsd/server/server.go b/x-pack/metricbeat/module/statsd/server/server.go index 48aee89e4608..c2366a71b6b8 100644 --- a/x-pack/metricbeat/module/statsd/server/server.go +++ b/x-pack/metricbeat/module/statsd/server/server.go @@ -95,7 +95,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { mappings, err := buildMappings(config.Mappings) if err != nil { - return nil, fmt.Errorf("invalid mapping configuration for `statsd.mapping`: %w", err) + return nil, fmt.Errorf("invalid mapping configuration for `statsd.mappings`: %w", err) } return &MetricSet{ BaseMetricSet: base, @@ -107,8 +107,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Host returns the hostname or other module specific value that identifies a // specific host or service instance from which to collect metrics. -func (b *MetricSet) Host() string { - return b.server.(*udp.UdpServer).GetHost() +func (m *MetricSet) Host() string { + return m.server.(*udp.UdpServer).GetHost() } func buildMappings(config []StatsdMapping) (map[string]StatsdMapping, error) { @@ -163,30 +163,36 @@ func buildMappings(config []StatsdMapping) (map[string]StatsdMapping, error) { return mappings, nil } +// It processes metric groups, applies event mappings, and creates Metricbeat events. +// The generated events include metric fields, labels, and the namespace associated with the MetricSet. +// Returns a slice of Metricbeat events. func (m *MetricSet) getEvents() []*mb.Event { groups := m.processor.GetAll() - events := make([]*mb.Event, len(groups)) - for idx, tagGroup := range groups { - - mapstrTags := mapstr.M{} + // If there are no metric groups, return nil to indicate no events. + if len(groups) == 0 { + return nil + } + events := make([]*mb.Event, 0, len(groups)) + for _, tagGroup := range groups { + mapstrTags := make(mapstr.M, len(tagGroup.tags)) for k, v := range tagGroup.tags { mapstrTags[k] = v } - sanitizedMetrics := mapstr.M{} for k, v := range tagGroup.metrics { - eventMapping(k, v, sanitizedMetrics, m.mappings) - } + // Apply event mapping to the metric and get MetricSetFields. + ms := eventMapping(k, v, m.mappings) - if len(sanitizedMetrics) == 0 { - continue - } - - events[idx] = &mb.Event{ - MetricSetFields: sanitizedMetrics, - RootFields: mapstr.M{"labels": mapstrTags}, - Namespace: m.Module().Name(), + // If no MetricSetFields were generated, continue to the next metric. + if len(ms) == 0 { + continue + } + events = append(events, &mb.Event{ + MetricSetFields: ms, + RootFields: mapstr.M{"labels": mapstrTags}, + Namespace: m.Module().Name(), + }) } } return events From e1aa97097a6e0acfbff77650859b7923db87b700 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 30 Oct 2023 12:32:23 -0400 Subject: [PATCH 42/50] chore: Update snapshot.yml (#36988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 28bb4bb0c7f8..7e210e414262 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-bb5042c3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-bb5042c3-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-fa958a90-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-bb5042c3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 069bf040d9689b77539b721dd8b1599b41cefd7c Mon Sep 17 00:00:00 2001 From: Christiano Haesbaert Date: Tue, 31 Oct 2023 08:52:42 +0100 Subject: [PATCH 43/50] Fix copy arguments for strict aligned architectures in ktrace decoding (#36976) Small typo. In practice only affects arm32 and arm64 as I doubt there are users of other architectures around. --- CHANGELOG-developer.next.asciidoc | 3 ++- CHANGELOG.next.asciidoc | 1 + x-pack/auditbeat/tracing/int_aligned.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 7fefb8d14c29..6a900dae8cf3 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -89,7 +89,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fixed some race conditions in tests {pull}36185[36185] - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525] - Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650] -- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] +- Cleaned up documentation errors & fixed a minor bug in Filebeat Azure blob storage input. {pull}36714[36714] +- Fix copy arguments for strict aligned architectures. {pull}36976[36976] ==== Added diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eef14a45f562..fd5543cb927a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -180,6 +180,7 @@ is collected by it. *Auditbeat* - Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851] +- Fix copy arguments for strict aligned architectures. {pull}36976[36976] *Filebeat* diff --git a/x-pack/auditbeat/tracing/int_aligned.go b/x-pack/auditbeat/tracing/int_aligned.go index 25c6094ef9a7..6c8c4c539725 100644 --- a/x-pack/auditbeat/tracing/int_aligned.go +++ b/x-pack/auditbeat/tracing/int_aligned.go @@ -16,7 +16,7 @@ import ( var errBadSize = errors.New("bad size for integer") func copyInt(dst unsafe.Pointer, src unsafe.Pointer, len uint8) error { - copy((*(*[maxIntSizeBytes]byte)(src))[:len], (*(*[maxIntSizeBytes]byte)(src))[:len]) + copy((*(*[maxIntSizeBytes]byte)(dst))[:len], (*(*[maxIntSizeBytes]byte)(src))[:len]) return nil } From a3ad99606854d4d31590fc49a77f02d21eedb09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Tue, 31 Oct 2023 08:58:24 +0100 Subject: [PATCH 44/50] Add state namespace metricset. (#36978) --- deploy/kubernetes/metricbeat-kubernetes.yaml | 1 + deploy/kubernetes/metricbeat/metricbeat-configmap.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index acece92b8d69..fa19c02043be 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -161,6 +161,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset diff --git a/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml index 388d4d7b550c..0cee98337983 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-configmap.yaml @@ -29,6 +29,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset From 009b3f0262d2ace24df0faf2ba1ae429708c107c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 31 Oct 2023 11:52:07 -0400 Subject: [PATCH 45/50] docs - Clarify how to use 'certificate' for servers (#36991) The docs didn't show where to put issuer certificate in the context of TLS servers. This explains how to build up a PEM certificate bundle --- libbeat/docs/shared-ssl-config.asciidoc | 98 +++++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index 1b27c3b217f8..3c456de42060 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -443,34 +443,84 @@ certificate_authorities: [[server-certificate]] ==== `certificate: "/etc/server/cert.pem"` -For server authentication, the path to the SSL authentication certificate must -be specified for TLS. If the certificate is not specified, startup will fail. +The end-entity (leaf) certificate that the server uses to identify itself. +If the certificate is signed by a certificate authority (CA), then it should +include intermediate CA certificates, sorted from leaf to root. +For servers, a `certificate` and <> must be specified. -When this option is configured, the <> option is also required. -The certificate option support embedding of the certificate: +The certificate option supports embedding of the PEM certificate content. This +example contains the leaf certificate followed by issuer's certificate. [source,yaml] ---- certificate: | - -----BEGIN CERTIFICATE----- - MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF - ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 - MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB - BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n - fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl - 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t - /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP - PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 - CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O - BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux - 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D - 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw - 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA - H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu - 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 - yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk - sxSmbIUfc2SGJGCJD4I= - -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIF2jCCA8KgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJVUzEW + MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8g + UmVhbDEOMAwGA1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwHhcNMjMxMDMw + MTkyMzU4WhcNMjMxMDMxMTkyMzU4WjB2MQswCQYDVQQGEwJVUzEWMBQGA1UEBxMN + U2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8gUmVhbDEOMAwG + A1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMxDzANBgNVBAMTBnNlcnZlcjCC + AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALW37cart7l0KE3LCStFbiGm + Rr/QSkuPv+Y+SXFT4zXrMFP3mOfUCVsR4lugv+jmql9qjbwR9jKsgKXA1kSvNXSZ + lLYWRcNnQ+QzwKxJf/jy246nSfqb2FKvVMs580lDwKHHxn/FSpHV93O4Goy5cLfF + ACE7BSdJdxl5DVAMmmkzd6gBGgN8dQIbcyJYuIZYQt44PqSYh/BomTyOXKrmvX4y + t7/pF+ldJjWZq/6SfCq6WE0jSrpI1P/42Qd9h5Tsnl6qsUGA2Tz5ZqKz2cyxaIlK + wL9tYDionfFIl+jZcxkGPF2a14O1TycCI0B/z+0VL+HR/8fKAB0NdP+QRLaPWOrn + DvraAO+bVKC6VrQyUYNUOwtd2gMUqm6Hzrf4s3wjP754eSJkvnSoSAB6l7ZmJKe5 + Pz5oDDOVPwKHv/MrhsCSMNFeXSEO+rq9TtYEAFQI5rFGHlURga8kA1T1pirHyEtS + 2o8GUSPSHVulaPdFnHg4xfTexfRYLCqya75ISJuY2/+2GblCie/re1GFitZCZ46/ + xiQQDOjgL96soDVZ+cTtMpXanslgDapTts9LPIJTd9FUJCY1omISGiSjABRuTlCV + 8054ja4BKVahSd5BqqtVkWyV64SCut6kce2ndwBkyFvlZ6cteLCW7KtzYvba4XBb + YIAs+H+9e/bZUVhws5mFAgMBAAGjgYMwgYAwDgYDVR0PAQH/BAQDAgeAMB0GA1Ud + JQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAOBgNVHQ4EBwQFAQIDBAUwPwYDVR0R + BDgwNoIJbG9jYWxob3N0ghFiZWF0cy5leGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAA + AAAAAAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAgEAldSZOUi+OUR46ERQuINl1oED + mjNsQ9FNP/RDu8mPJaNb5v2sAbcpuZb9YdnScT+d+n0+LMd5uz2g67Qr73QCpXwL + 9YJIs56i7qMTKXlVvRQrvF9P/zP3sm5Zfd2I/x+8oXgEeYsxAWipJ8RsbnN1dtu8 + C4l+P0E58jjrjom11W90RiHYaT0SI2PPBTTRhYLz0HayThPZDMdFnIQqVxUYbQD5 + ybWu77hnsvC/g2C8/N2LAdQGJJ67owMa5T3YRneiaSvvOf3I45oeLE+olGAPdrSq + 5Sp0G7fcAKMRPxcwYeD7V5lfYMtb+RzECpYAHT8zHKLZl6/34q2k8P8EWEpAsD80 + +zSbCkdvNiU5lU90rV8E2baTKCg871k4O8sT48eUyDps6ZUCfT1dgefXeyOTV5bY + 864Zo6bWJhAJ7Qa2d4HJkqPzSbqsosHVobojgkOcMqkStLHd8sgtCoFmJMflbp7E + ghawl/RVFEkL9+TWy9fR8sJWRx13P8CUP6AL9kVmcU2c3gMNpvQfIii9QOnQrRsi + yZj9FKl+ZM49I6RQ6dY5JVgWtpVm/+GBVuy1Aj91JEjw7r1jAeir5K9LAXG8kEN9 + irndx1SK2MMTY79lGHFGQRv3vnQGI0Wzjtn31YJ7qIFNJ1WWbAZLR9FBtzmMeXM6 + puoJ9UYvfIcHUGPdZGU= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIFpjCCA46gAwIBAgIBATANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJVUzEW + MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8g + UmVhbDEOMAwGA1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwHhcNMjMxMDMw + MTkyMzU2WhcNMjMxMDMxMTkyMzU2WjBlMQswCQYDVQQGEwJVUzEWMBQGA1UEBxMN + U2FuIEZyYW5jaXNjbzEcMBoGA1UECRMTV2VzdCBFbCBDYW1pbm8gUmVhbDEOMAwG + A1UEERMFOTQwNDAxEDAOBgNVBAoTB0VsYXN0aWMwggIiMA0GCSqGSIb3DQEBAQUA + A4ICDwAwggIKAoICAQDQP3hJt4jTIo+tBXB/R4RuBTvv6OOago9joxlNDm0abseJ + ehE0V8FDi0SSpa7ZiqwCGq/deu5OIWVNpFCLHeH5YBriNmB7oPkNRCleu50JsUrG + RjSTtBIJcu/CVpD7Q5XMbhbhYcPArrxrSreo3ox8a+2X7b8nA1xPgIcWqSCgs9iV + lwKHaQWNTUXYwwZG7b9WG4EJaki6t1+1QbDDJU0oWrZNg23wQEBvEVRDQs7kadvm + 9YtZLPULlSyV4Rk3yNW8dPXHjcz2wp3PBPIWIQe9mzYU608307TkUMVN2EEOImxl + Wm1RtXYvvVb1LiY0C2lYbN3jLZQzffK5RsS87ocqTQM+HvDBv/PupHDvW08wietu + RtRbdx/2cN0GLmOHnkWKx+GlYDZfAtIj958fTKl2hHyNqJ1pE7vksSYBwBxMFQem + eSGzw5pO53kmPcZO203YQ2qoJd7z1aLf7eAOqDn5zwlYNc00bZ6DwTZsyptGv9sZ + zcZuovppPgCN4f1I9ja/NPKep+sVKfQqR5HuOFOPFcr6oOioESJSgIvXXF9RhCVh + UMeZKWWSCNm1ea4h6q8OJdQfM7XXkXm+dEyF0TogC00CidZWuYMZcgXND5p/1Di5 + PkCKPUMllCoK0oaTfFioNW7qtNbDGQrW+spwDa4kjJNKYtDD0jjPgFMgSzQ2MwID + AQABo2EwXzAOBgNVHQ8BAf8EBAMCAoQwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsG + AQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFImOXc9Tv+mgn9jOsPig + 9vlAUTa+MA0GCSqGSIb3DQEBCwUAA4ICAQBZ9tqU88Nmgf+vDgkKMKLmLMaRCRlV + HcYrm7WoWLX+q6VSbmvf5eD5OrzzbAnnp16iXap8ivsAEFTo8XWh/bjl7G/2jetR + xZD2WHtzmAg3s4SVsEHIyFUF1ERwnjO2ndHjoIsx8ktUk1aNrmgPI6s07fkULDm+ + 2aXyBSZ9/oimZM/s3IqYJecxwE+yyS+FiS6mSDCCVIyQXdtVAbFHegyiBYv8EbwF + Xz70QiqQtxotGlfts/3uN1s+xnEoWz5E6S5DQn4xQh0xiKSXPizMXou9xKzypeSW + qtNdwtg62jKWDaVriBfrvoCnyjjCIjmcTcvA2VLmeZShyTuIucd0lkg2NKIGeM7I + o33hmdiKaop1fVtj8zqXvCRa3ecmlvcxPKX0otVFORFNOfaPjH/CjW0CnP0LByGK + YW19w0ncJZa9cc1SlNL28lnBhW+i1+ViR02wtjabH9XO+mtxuaEPDZ1hLhhjktqI + Y2oFUso4C5xiTU/hrH8+cFv0dn/+zyQoLfJEQbUX9biFeytt7T4Yynwhdy7jryqH + fdy/QM26YnsE8D7l4mv99z+zII0IRGnQOuLTuNAIyGJUf69hCDubZFDeHV/IB9hU + 6GA6lBpsJlTDgfJLbtKuAHxdn1DO+uGg0GxgwggH6Vh9x9yQK2E6BaepJisL/zNB + RQQmEyTn1hn/eA== + -----END CERTIFICATE----- ---- [float] @@ -478,7 +528,7 @@ certificate: | ==== `key: "/etc/server/cert.key"` The server certificate key used for authentication is required. -The key option support embedding of the private key: +The key option supports embedding of the private key: [source,yaml] ---- From 2539cdc2577b7aad659f78ea670308a7d38dfe13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Tue, 31 Oct 2023 19:47:16 +0100 Subject: [PATCH 46/50] [Heartbeat] Bump nodejs minor version (#36961) Bump NodeJS version bundled with Heartbeat to 18.18.2. Co-authored-by: Tiago Queiroz --------- Co-authored-by: Tiago Queiroz --- CHANGELOG.next.asciidoc | 1 + dev-tools/packaging/templates/docker/Dockerfile.tmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index fd5543cb927a..2d2364180d00 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -119,6 +119,7 @@ is collected by it. - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix retries to trigger on a down monitor with no previous state. {pull}36842[36842] +- Bump NodeJS minor version to 18.18.2. {pull}36961[36961] - Fix monitor duration calculation with retries. {pull}36900[36900] *Metricbeat* diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 305687d33e01..9a792ecc1e0a 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -123,7 +123,7 @@ RUN echo \ ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV TZ=UTC ENV SUITES_DIR={{ $beatHome }}/suites -ENV NODE_VERSION=18.16.0 +ENV NODE_VERSION=18.18.2 ENV PATH="$NODE_PATH/node/bin:$PATH" # Install the latest version of @elastic/synthetics forcefully ignoring the previously # cached node_modules, heartbeat then calls the global executable to run test suites From 32cdaf67df53913568afc290c922c83e3154c960 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 31 Oct 2023 16:11:32 -0400 Subject: [PATCH 47/50] chore: Update snapshot.yml (#37003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 7e210e414262..cb047a327df7 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-0fcb9daf-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-0fcb9daf-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-bb5042c3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-0fcb9daf-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From c0a647add4f1f88f115b00516e8e3a379ac17cee Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Wed, 1 Nov 2023 14:51:08 +1030 Subject: [PATCH 48/50] x-pack/winlogbeat/modules/security: fix UAC attribute bit table (#37009) The previous table was incorrect. Table data comes from MS-SAMR: Security Account Manager (SAM) Remote Protocol (Client-to-Server) version 46.0[1], 2.2.1.12 USER_ACCOUNT Codes. [1]https://winprotocoldoc.blob.core.windows.net/productionwindowsarchives/MS-SAMR/%5bMS-SAMR%5d-230828.docx --- CHANGELOG.next.asciidoc | 1 + .../module/security/ingest/security.yml | 46 +++++++++---------- .../test/testdata/ingest/1100.golden.json | 1 - .../test/testdata/ingest/1102.golden.json | 1 - .../test/testdata/ingest/1104.golden.json | 1 - .../test/testdata/ingest/1105.golden.json | 1 - .../ingest/4670_WindowsSrv2016.golden.json | 1 - .../ingest/4706_WindowsSrv2016.golden.json | 1 - .../ingest/4707_WindowsSrv2016.golden.json | 1 - .../ingest/4713_WindowsSrv2016.golden.json | 1 - .../ingest/4716_WindowsSrv2016.golden.json | 1 - .../ingest/4717_WindowsSrv2016.golden.json | 1 - .../ingest/4718_WindowsSrv2016.golden.json | 1 - .../test/testdata/ingest/4719.golden.json | 1 - .../ingest/4719_WindowsSrv2016.golden.json | 1 - .../ingest/4739_WindowsSrv2016.golden.json | 1 - .../test/testdata/ingest/4741.golden.json | 6 +-- .../test/testdata/ingest/4742.golden.json | 4 +- .../test/testdata/ingest/4743.golden.json | 1 - .../test/testdata/ingest/4744.golden.json | 1 - .../test/testdata/ingest/4745.golden.json | 1 - .../test/testdata/ingest/4746.golden.json | 1 - .../test/testdata/ingest/4747.golden.json | 1 - .../test/testdata/ingest/4748.golden.json | 1 - .../test/testdata/ingest/4749.golden.json | 1 - .../test/testdata/ingest/4750.golden.json | 1 - .../test/testdata/ingest/4751.golden.json | 1 - .../test/testdata/ingest/4752.golden.json | 1 - .../test/testdata/ingest/4753.golden.json | 1 - .../test/testdata/ingest/4759.golden.json | 1 - .../test/testdata/ingest/4760.golden.json | 1 - .../test/testdata/ingest/4761.golden.json | 1 - .../test/testdata/ingest/4762.golden.json | 1 - .../test/testdata/ingest/4763.golden.json | 1 - .../ingest/4817_WindowsSrv2016.golden.json | 1 - .../ingest/4902_WindowsSrv2016.golden.json | 1 - .../ingest/4904_WindowsSrv2016.golden.json | 1 - .../ingest/4905_WindowsSrv2016.golden.json | 1 - .../ingest/4906_WindowsSrv2016.golden.json | 1 - .../ingest/4907_WindowsSrv2016.golden.json | 1 - .../ingest/4908_WindowsSrv2016.golden.json | 1 - .../security-windows2012_4673.golden.json | 1 - .../security-windows2012_4674.golden.json | 1 - .../security-windows2012_4697.golden.json | 1 - .../security-windows2012_4698.golden.json | 1 - .../security-windows2012_4699.golden.json | 1 - .../security-windows2012_4700.golden.json | 1 - .../security-windows2012_4701.golden.json | 1 - .../security-windows2012_4702.golden.json | 1 - .../security-windows2012_4768.golden.json | 1 - .../security-windows2012_4769.golden.json | 1 - .../security-windows2012_4770.golden.json | 1 - .../security-windows2012_4771.golden.json | 1 - .../security-windows2012_4776.golden.json | 1 - .../security-windows2012_4778.golden.json | 1 - .../security-windows2012_4779.golden.json | 1 - .../security-windows2012r2-logon.golden.json | 18 -------- .../security-windows2016-4672.golden.json | 1 - .../security-windows2016-logoff.golden.json | 2 - ...ndows2016_4720_Account_Created.golden.json | 12 ++--- ...ndows2016_4722_Account_Enabled.golden.json | 2 - ...ndows2016_4723_Password_Change.golden.json | 2 - ...indows2016_4724_Password_Reset.golden.json | 2 - ...dows2016_4725_Account_Disabled.golden.json | 2 - ...ndows2016_4726_Account_Deleted.golden.json | 2 - .../security-windows2016_4727.golden.json | 1 - .../security-windows2016_4728.golden.json | 1 - .../security-windows2016_4729.golden.json | 1 - .../security-windows2016_4730.golden.json | 1 - .../security-windows2016_4731.golden.json | 1 - .../security-windows2016_4732.golden.json | 1 - .../security-windows2016_4733.golden.json | 1 - .../security-windows2016_4734.golden.json | 1 - .../security-windows2016_4735.golden.json | 1 - .../security-windows2016_4737.golden.json | 1 - ...ndows2016_4738_Account_Changed.golden.json | 10 ++-- ...ws2016_4740_Account_Locked_Out.golden.json | 1 - .../security-windows2016_4754.golden.json | 1 - .../security-windows2016_4755.golden.json | 1 - .../security-windows2016_4756.golden.json | 1 - .../security-windows2016_4757.golden.json | 1 - .../security-windows2016_4758.golden.json | 1 - .../security-windows2016_4764.golden.json | 1 - ...dows2016_4767_Account_Unlocked.golden.json | 1 - ...ndows2016_4781_Account_Renamed.golden.json | 2 - .../security-windows2016_4798.golden.json | 1 - .../security-windows2016_4799.golden.json | 1 - .../security-windows2016_4964.golden.json | 2 - ...ndows2019_4688_Process_Created.golden.json | 1 - ...indows2019_4689_Process_Exited.golden.json | 3 -- 90 files changed, 39 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2d2364180d00..3fc9493640b3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -154,6 +154,7 @@ is collected by it. *Winlogbeat* +- Fix User Account Control Attributes Table values for Security module. {issue}36999[36999] {pull}37009[37009] *Elastic Logging Plugin* diff --git a/x-pack/winlogbeat/module/security/ingest/security.yml b/x-pack/winlogbeat/module/security/ingest/security.yml index 846fdd26fbbe..020b14af356b 100644 --- a/x-pack/winlogbeat/module/security/ingest/security.yml +++ b/x-pack/winlogbeat/module/security/ingest/security.yml @@ -836,30 +836,30 @@ processors: tag: Set User Account Control description: Set User Account Control # User Account Control Attributes Table - # https://support.microsoft.com/es-us/help/305144/how-to-use-useraccountcontrol-to-manipulate-user-account-properties + # https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-samr/4df07fab-1bbc-452f-8e92-7853a3c7e380 params: - "0x00000001": SCRIPT - "0x00000002": ACCOUNTDISABLE - "0x00000008": HOMEDIR_REQUIRED - "0x00000010": LOCKOUT - "0x00000020": PASSWD_NOTREQD - "0x00000040": PASSWD_CANT_CHANGE - "0x00000080": ENCRYPTED_TEXT_PWD_ALLOWED - "0x00000100": TEMP_DUPLICATE_ACCOUNT - "0x00000200": NORMAL_ACCOUNT - "0x00000800": INTERDOMAIN_TRUST_ACCOUNT - "0x00001000": WORKSTATION_TRUST_ACCOUNT - "0x00002000": SERVER_TRUST_ACCOUNT - "0x00010000": DONT_EXPIRE_PASSWORD - "0x00020000": MNS_LOGON_ACCOUNT - "0x00040000": SMARTCARD_REQUIRED - "0x00080000": TRUSTED_FOR_DELEGATION - "0x00100000": NOT_DELEGATED - "0x00200000": USE_DES_KEY_ONLY - "0x00400000": DONT_REQ_PREAUTH - "0x00800000": PASSWORD_EXPIRED - "0x01000000": TRUSTED_TO_AUTH_FOR_DELEGATION - "0x04000000": PARTIAL_SECRETS_ACCOUNT + "0x00000001": USER_ACCOUNT_DISABLED + "0x00000002": USER_HOME_DIRECTORY_REQUIRED + "0x00000004": USER_PASSWORD_NOT_REQUIRED + "0x00000008": USER_TEMP_DUPLICATE_ACCOUNT + "0x00000010": USER_NORMAL_ACCOUNT + "0x00000020": USER_MNS_LOGON_ACCOUNT + "0x00000040": USER_INTERDOMAIN_TRUST_ACCOUNT + "0x00000080": USER_WORKSTATION_TRUST_ACCOUNT + "0x00000100": USER_SERVER_TRUST_ACCOUNT + "0x00000200": USER_DONT_EXPIRE_PASSWORD + "0x00000400": USER_ACCOUNT_AUTO_LOCKED + "0x00000800": USER_ENCRYPTED_TEXT_PASSWORD_ALLOWED + "0x00001000": USER_SMARTCARD_REQUIRED + "0x00002000": USER_TRUSTED_FOR_DELEGATION + "0x00004000": USER_NOT_DELEGATED + "0x00008000": USER_USE_DES_KEY_ONLY + "0x00010000": USER_DONT_REQUIRE_PREAUTH + "0x00020000": USER_PASSWORD_EXPIRED + "0x00040000": USER_TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION + "0x00080000": USER_NO_AUTH_DATA_REQUIRED + "0x00100000": USER_PARTIAL_SECRETS_ACCOUNT + "0x00200000": USER_USE_AES_KEYS source: |- if (ctx?.winlog?.event_data?.NewUacValue == null) { return; diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/1100.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/1100.golden.json index 1eaf254cb9cd..0e6fdfaa198b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/1100.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/1100.golden.json @@ -10,7 +10,6 @@ "process" ], "code": "1100", - "ingested": "2022-06-08T06:21:07.784686200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/1102.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/1102.golden.json index 6374f10e8eb5..f25bf0530d41 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/1102.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/1102.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "1102", - "ingested": "2022-06-08T06:21:07.838072400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/1104.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/1104.golden.json index d54a6ee27af3..72250f42747b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/1104.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/1104.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "1104", - "ingested": "2022-06-08T06:21:07.850785400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/1105.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/1105.golden.json index 066a1ba598d3..0fd4278b24f9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/1105.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/1105.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "1105", - "ingested": "2022-06-08T06:21:07.856253Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4670_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4670_WindowsSrv2016.golden.json index 0ac7449263be..3896d299bcae 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4670_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4670_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4670", - "ingested": "2022-06-08T06:21:07.861752100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4706_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4706_WindowsSrv2016.golden.json index 7d98f44725cf..08d1e9e963bb 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4706_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4706_WindowsSrv2016.golden.json @@ -10,7 +10,6 @@ "configuration" ], "code": "4706", - "ingested": "2022-06-08T06:21:07.908218700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4707_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4707_WindowsSrv2016.golden.json index ab4a62ab9b2f..55b0f03274f1 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4707_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4707_WindowsSrv2016.golden.json @@ -10,7 +10,6 @@ "configuration" ], "code": "4707", - "ingested": "2022-06-08T06:21:07.915673700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4713_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4713_WindowsSrv2016.golden.json index 7a8930ce885f..6e14de618248 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4713_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4713_WindowsSrv2016.golden.json @@ -10,7 +10,6 @@ "configuration" ], "code": "4713", - "ingested": "2022-06-08T06:21:07.921167700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4716_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4716_WindowsSrv2016.golden.json index 57c656a76707..e11018510a4f 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4716_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4716_WindowsSrv2016.golden.json @@ -10,7 +10,6 @@ "configuration" ], "code": "4716", - "ingested": "2022-06-08T06:21:07.926829100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4717_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4717_WindowsSrv2016.golden.json index 30a0da980a23..231a35064e55 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4717_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4717_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4717", - "ingested": "2022-06-08T06:21:07.932459300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4718_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4718_WindowsSrv2016.golden.json index 3becc27b8f20..48cb2f747853 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4718_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4718_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4718", - "ingested": "2022-06-08T06:21:07.938661600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4719.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4719.golden.json index b43487b6efb4..b034d8caa038 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4719.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4719.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4719", - "ingested": "2022-06-08T06:21:07.944221400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4719_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4719_WindowsSrv2016.golden.json index 92e60c91e1d6..770bb463c1a5 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4719_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4719_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4719", - "ingested": "2022-06-08T06:21:07.955823800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4739_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4739_WindowsSrv2016.golden.json index b7a566a0fffd..04b96ac55831 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4739_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4739_WindowsSrv2016.golden.json @@ -10,7 +10,6 @@ "configuration" ], "code": "4739", - "ingested": "2022-06-08T06:21:07.963089600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4741.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4741.golden.json index 1ae1d6e9de9f..b2451c6b75bf 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4741.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4741.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4741", - "ingested": "2022-06-08T06:21:07.970367200Z", "kind": "event", "module": "security", "outcome": "success", @@ -55,8 +54,9 @@ "HomePath": "-", "LogonHours": "%%1793", "NewUACList": [ - "SCRIPT", - "ENCRYPTED_TEXT_PWD_ALLOWED" + "USER_ACCOUNT_DISABLED", + "USER_PASSWORD_NOT_REQUIRED", + "USER_WORKSTATION_TRUST_ACCOUNT" ], "NewUacValue": "0x85", "OldUacValue": "0x0", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4742.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4742.golden.json index 6eb53747422c..4233d2fec7a8 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4742.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4742.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4742", - "ingested": "2022-06-08T06:21:07.984310900Z", "kind": "event", "module": "security", "outcome": "success", @@ -56,7 +55,8 @@ "HomePath": "-", "LogonHours": "-", "NewUACList": [ - "ENCRYPTED_TEXT_PWD_ALLOWED" + "USER_PASSWORD_NOT_REQUIRED", + "USER_WORKSTATION_TRUST_ACCOUNT" ], "NewUacValue": "0x84", "OldUacValue": "0x85", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4743.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4743.golden.json index 29f38474c35d..b7cd00dd8bfd 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4743.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4743.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4743", - "ingested": "2022-06-08T06:21:07.989281200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4744.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4744.golden.json index 635787f0a4a3..14340f1898d9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4744.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4744.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4744", - "ingested": "2022-06-08T06:21:07.994556700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4745.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4745.golden.json index eeee7ce2fc9e..cb0b005fc8e9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4745.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4745.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4745", - "ingested": "2022-06-08T06:21:08.002640900Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4746.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4746.golden.json index db2cde52acd9..8922fa5b1d65 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4746.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4746.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4746", - "ingested": "2022-06-08T06:21:08.017662600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4747.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4747.golden.json index b1d1db2e16af..5e85e85cb347 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4747.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4747.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4747", - "ingested": "2022-06-08T06:21:08.025768800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4748.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4748.golden.json index bc74a2fdf8b4..b3212145129d 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4748.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4748.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4748", - "ingested": "2022-06-08T06:21:08.030353100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4749.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4749.golden.json index bc107b8485b2..a85446eede79 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4749.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4749.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4749", - "ingested": "2022-06-08T06:21:08.034749600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4750.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4750.golden.json index 5f6bdc8c532b..f66b9e93c991 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4750.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4750.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4750", - "ingested": "2022-06-08T06:21:08.039233400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4751.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4751.golden.json index 47f8fc9c650a..981b0e942994 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4751.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4751.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4751", - "ingested": "2022-06-08T06:21:08.051295Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4752.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4752.golden.json index 24b12c361f93..1aae64679089 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4752.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4752.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4752", - "ingested": "2022-06-08T06:21:08.057508500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4753.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4753.golden.json index df72711f8062..46d6bb0e394b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4753.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4753.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4753", - "ingested": "2022-06-08T06:21:08.063346200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4759.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4759.golden.json index 7c62dac0da7b..ba25270a4c80 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4759.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4759.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4759", - "ingested": "2022-06-08T06:21:08.069524100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4760.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4760.golden.json index f92c3a95d657..d316a4133ec8 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4760.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4760.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4760", - "ingested": "2022-06-08T06:21:08.074975800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4761.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4761.golden.json index 8c22b8f7d591..9c37bd371f07 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4761.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4761.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4761", - "ingested": "2022-06-08T06:21:08.080868Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4762.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4762.golden.json index e088f163bc89..4bfe930b0da3 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4762.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4762.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4762", - "ingested": "2022-06-08T06:21:08.086379300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4763.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4763.golden.json index 1a0a51efac0b..780b65bd7ef3 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4763.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4763.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4763", - "ingested": "2022-06-08T06:21:08.092821300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4817_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4817_WindowsSrv2016.golden.json index 8917f0c86ec2..cb3a98d3882c 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4817_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4817_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4817", - "ingested": "2022-06-08T06:21:08.101661100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4902_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4902_WindowsSrv2016.golden.json index bab3c88b7951..5c6dab0f41dc 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4902_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4902_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4902", - "ingested": "2022-06-08T06:21:08.110215500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4904_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4904_WindowsSrv2016.golden.json index b253a2460136..c91fd476614c 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4904_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4904_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4904", - "ingested": "2022-06-08T06:21:08.115118100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4905_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4905_WindowsSrv2016.golden.json index 38daa55319b4..4eb0139038c2 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4905_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4905_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4905", - "ingested": "2022-06-08T06:21:08.119957100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4906_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4906_WindowsSrv2016.golden.json index 742d95031e5c..2e71ca0361d6 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4906_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4906_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4906", - "ingested": "2022-06-08T06:21:08.124490200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4907_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4907_WindowsSrv2016.golden.json index 1010ad2b281c..8ec45c3dbd8a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4907_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4907_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4907", - "ingested": "2022-06-08T06:21:08.129757100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/4908_WindowsSrv2016.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/4908_WindowsSrv2016.golden.json index 7fcc0d935f56..9acba8df75c4 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/4908_WindowsSrv2016.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/4908_WindowsSrv2016.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4908", - "ingested": "2022-06-09T04:25:10.390738Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4673.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4673.golden.json index ee3d3ecca909..b1724e0f4c63 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4673.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4673.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4673", - "ingested": "2022-06-08T06:21:08.143556300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4674.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4674.golden.json index 8a5984909503..15e95215432b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4674.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4674.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4674", - "ingested": "2022-06-08T06:21:08.149617100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4697.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4697.golden.json index 2364c9c945a3..4583f47bf551 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4697.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4697.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4697", - "ingested": "2022-06-08T06:21:08.156998700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4698.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4698.golden.json index 16ac2d25a4c8..cb07d880e629 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4698.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4698.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4698", - "ingested": "2022-06-08T06:21:08.162693600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4699.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4699.golden.json index c467fe970561..b3c26a4f56ad 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4699.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4699.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4699", - "ingested": "2022-06-08T06:21:08.168246300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4700.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4700.golden.json index 8fee3ad99d1b..8ad5f4600d3a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4700.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4700.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4700", - "ingested": "2022-06-08T06:21:08.173701300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4701.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4701.golden.json index 62b7e37c3bb6..cc62f8e6c6eb 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4701.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4701.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4701", - "ingested": "2022-06-08T06:21:08.178899800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4702.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4702.golden.json index 6525f91ebfd2..2352de8c4945 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4702.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4702.golden.json @@ -11,7 +11,6 @@ "configuration" ], "code": "4702", - "ingested": "2022-06-08T06:21:08.184651500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4768.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4768.golden.json index f3740d6b1cf6..c1c40241415d 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4768.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4768.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4768", - "ingested": "2022-06-08T06:21:08.190661800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4769.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4769.golden.json index 8af6ff6551c0..ddf98ed0b7cd 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4769.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4769.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4769", - "ingested": "2022-06-08T06:21:08.199357500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4770.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4770.golden.json index 41b1b2aee08a..d76e139ac7a0 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4770.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4770.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4770", - "ingested": "2022-06-08T06:21:08.204255500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4771.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4771.golden.json index 27c8e247554e..8ac4835eb362 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4771.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4771.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4771", - "ingested": "2022-06-08T06:21:08.209027300Z", "kind": "event", "module": "security", "outcome": "failure", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4776.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4776.golden.json index 2e6296b41231..e9ee05ee05c4 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4776.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4776.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4776", - "ingested": "2022-06-08T06:21:08.213859200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4778.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4778.golden.json index 19127eb1267e..da716047cdd2 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4778.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4778.golden.json @@ -11,7 +11,6 @@ "session" ], "code": "4778", - "ingested": "2022-06-08T06:21:08.218876200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4779.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4779.golden.json index aabbb2b28434..fb96fcfc2a69 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4779.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012_4779.golden.json @@ -11,7 +11,6 @@ "session" ], "code": "4779", - "ingested": "2022-06-08T06:21:08.225133500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012r2-logon.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012r2-logon.golden.json index 9fa6d71e344a..ca2e383e6001 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012r2-logon.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2012r2-logon.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233175700Z", "kind": "event", "module": "security", "outcome": "success", @@ -99,7 +98,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233202400Z", "kind": "event", "module": "security", "outcome": "success", @@ -188,7 +186,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233212800Z", "kind": "event", "module": "security", "outcome": "success", @@ -283,7 +280,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233221700Z", "kind": "event", "module": "security", "outcome": "success", @@ -372,7 +368,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233234500Z", "kind": "event", "module": "security", "outcome": "success", @@ -460,7 +455,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233247200Z", "kind": "event", "module": "security", "outcome": "success", @@ -548,7 +542,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233259700Z", "kind": "event", "module": "security", "outcome": "success", @@ -636,7 +629,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233272300Z", "kind": "event", "module": "security", "outcome": "success", @@ -724,7 +716,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233284800Z", "kind": "event", "module": "security", "outcome": "success", @@ -815,7 +806,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233299300Z", "kind": "event", "module": "security", "outcome": "success", @@ -904,7 +894,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233312100Z", "kind": "event", "module": "security", "outcome": "success", @@ -999,7 +988,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233322700Z", "kind": "event", "module": "security", "outcome": "success", @@ -1088,7 +1076,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233331Z", "kind": "event", "module": "security", "outcome": "success", @@ -1177,7 +1164,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233343700Z", "kind": "event", "module": "security", "outcome": "success", @@ -1266,7 +1252,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233356Z", "kind": "event", "module": "security", "outcome": "success", @@ -1355,7 +1340,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233386300Z", "kind": "event", "module": "security", "outcome": "success", @@ -1444,7 +1428,6 @@ "authentication" ], "code": "4624", - "ingested": "2022-06-08T06:21:08.233400100Z", "kind": "event", "module": "security", "outcome": "success", @@ -1533,7 +1516,6 @@ "authentication" ], "code": "4625", - "ingested": "2022-06-08T06:21:08.233408800Z", "kind": "event", "module": "security", "outcome": "failure", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-4672.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-4672.golden.json index bdb665abba4e..a4d6dd612030 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-4672.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-4672.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4672", - "ingested": "2022-06-08T06:21:08.268186400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-logoff.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-logoff.golden.json index 278965f26e12..a7fcd4fe171e 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-logoff.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016-logoff.golden.json @@ -10,7 +10,6 @@ "authentication" ], "code": "4634", - "ingested": "2022-06-08T06:21:08.274378300Z", "kind": "event", "module": "security", "outcome": "success", @@ -79,7 +78,6 @@ "authentication" ], "code": "4634", - "ingested": "2022-06-08T06:21:08.274397100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4720_Account_Created.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4720_Account_Created.golden.json index 42d00562adbc..37cfbae0c189 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4720_Account_Created.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4720_Account_Created.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4720", - "ingested": "2022-06-08T06:21:08.280835800Z", "kind": "event", "module": "security", "outcome": "success", @@ -56,8 +55,9 @@ "HomePath": "%%1793", "LogonHours": "%%1797", "NewUACList": [ - "SCRIPT", - "LOCKOUT" + "USER_ACCOUNT_DISABLED", + "USER_PASSWORD_NOT_REQUIRED", + "USER_NORMAL_ACCOUNT" ], "NewUacValue": "0x15", "OldUacValue": "0x0", @@ -115,7 +115,6 @@ "iam" ], "code": "4720", - "ingested": "2022-06-08T06:21:08.280855300Z", "kind": "event", "module": "security", "outcome": "success", @@ -161,8 +160,9 @@ "HomePath": "%%1793", "LogonHours": "%%1797", "NewUACList": [ - "SCRIPT", - "LOCKOUT" + "USER_ACCOUNT_DISABLED", + "USER_PASSWORD_NOT_REQUIRED", + "USER_NORMAL_ACCOUNT" ], "NewUacValue": "0x15", "OldUacValue": "0x0", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4722_Account_Enabled.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4722_Account_Enabled.golden.json index 4309e4ab5630..8acd38ccd8b9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4722_Account_Enabled.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4722_Account_Enabled.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4722", - "ingested": "2022-06-08T06:21:08.289118900Z", "kind": "event", "module": "security", "outcome": "success", @@ -88,7 +87,6 @@ "iam" ], "code": "4722", - "ingested": "2022-06-08T06:21:08.289135100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4723_Password_Change.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4723_Password_Change.golden.json index 78be6924e116..0738a3cac0c8 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4723_Password_Change.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4723_Password_Change.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4723", - "ingested": "2022-06-08T06:21:08.294838400Z", "kind": "event", "module": "security", "outcome": "failure", @@ -88,7 +87,6 @@ "iam" ], "code": "4723", - "ingested": "2022-06-08T06:21:08.294849500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4724_Password_Reset.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4724_Password_Reset.golden.json index 1222c6ba73ec..58663b216186 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4724_Password_Reset.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4724_Password_Reset.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4724", - "ingested": "2022-06-08T06:21:08.301226700Z", "kind": "event", "module": "security", "outcome": "success", @@ -88,7 +87,6 @@ "iam" ], "code": "4724", - "ingested": "2022-06-08T06:21:08.301245800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4725_Account_Disabled.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4725_Account_Disabled.golden.json index 1ef20e9827b7..5b50072792db 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4725_Account_Disabled.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4725_Account_Disabled.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4725", - "ingested": "2022-06-08T06:21:08.307262100Z", "kind": "event", "module": "security", "outcome": "success", @@ -88,7 +87,6 @@ "iam" ], "code": "4725", - "ingested": "2022-06-08T06:21:08.307282600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4726_Account_Deleted.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4726_Account_Deleted.golden.json index 7ae020ab4103..9b8a6c9fc01b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4726_Account_Deleted.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4726_Account_Deleted.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4726", - "ingested": "2022-06-08T06:21:08.313870400Z", "kind": "event", "module": "security", "outcome": "success", @@ -89,7 +88,6 @@ "iam" ], "code": "4726", - "ingested": "2022-06-08T06:21:08.313890100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4727.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4727.golden.json index 5f769bd55805..d1779d672d1f 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4727.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4727.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4727", - "ingested": "2022-06-08T06:21:08.321314400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4728.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4728.golden.json index 7b84dca06396..1c5e5f81d93c 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4728.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4728.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4728", - "ingested": "2022-06-08T06:21:08.326362Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4729.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4729.golden.json index 143098d35b54..836f496a6344 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4729.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4729.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4729", - "ingested": "2022-06-08T06:21:08.331614100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4730.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4730.golden.json index d49edf637285..951030babe63 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4730.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4730.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4730", - "ingested": "2022-06-08T06:21:08.335699200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4731.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4731.golden.json index de17dbfd0f30..2c2d9988fb59 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4731.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4731.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4731", - "ingested": "2022-06-08T06:21:08.339943300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4732.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4732.golden.json index c4309ee4d91d..8d382bc8cc6e 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4732.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4732.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4732", - "ingested": "2022-06-08T06:21:08.344534800Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4733.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4733.golden.json index 17f9624127af..5bc1bdbe2d59 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4733.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4733.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4733", - "ingested": "2022-06-08T06:21:08.349155Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4734.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4734.golden.json index 225dcd822b51..2042cc2b9323 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4734.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4734.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4734", - "ingested": "2022-06-08T06:21:08.353949600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4735.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4735.golden.json index 4c8ed6942289..4ad378e3040a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4735.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4735.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4735", - "ingested": "2022-06-08T06:21:08.358858100Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4737.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4737.golden.json index 31ca8e5aa0d9..7fdf3fb4589b 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4737.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4737.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4737", - "ingested": "2022-06-08T06:21:08.363652300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4738_Account_Changed.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4738_Account_Changed.golden.json index c696e8495fee..025b1f20bca4 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4738_Account_Changed.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4738_Account_Changed.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4738", - "ingested": "2022-06-08T06:21:08.368099900Z", "kind": "event", "module": "security", "outcome": "success", @@ -57,8 +56,8 @@ "HomePath": "%%1793", "LogonHours": "%%1797", "NewUACList": [ - "LOCKOUT", - "NORMAL_ACCOUNT" + "USER_NORMAL_ACCOUNT", + "USER_DONT_EXPIRE_PASSWORD" ], "NewUacValue": "0x210", "OldUacValue": "0x210", @@ -114,7 +113,6 @@ "iam" ], "code": "4738", - "ingested": "2022-06-08T06:21:08.368122700Z", "kind": "event", "module": "security", "outcome": "success", @@ -161,8 +159,8 @@ "HomePath": "%%1793", "LogonHours": "%%1797", "NewUACList": [ - "LOCKOUT", - "NORMAL_ACCOUNT" + "USER_NORMAL_ACCOUNT", + "USER_DONT_EXPIRE_PASSWORD" ], "NewUacValue": "0x210", "OldUacValue": "0x10", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4740_Account_Locked_Out.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4740_Account_Locked_Out.golden.json index f9adb19e87ff..eaec5eb7104f 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4740_Account_Locked_Out.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4740_Account_Locked_Out.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4740", - "ingested": "2022-06-08T06:21:08.374783400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4754.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4754.golden.json index f59e7492bb51..f01b564e6ec9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4754.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4754.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4754", - "ingested": "2022-06-08T06:21:08.378911400Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4755.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4755.golden.json index b2481b95d8f0..7d41c2ec98d2 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4755.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4755.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4755", - "ingested": "2022-06-08T06:21:08.383281700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4756.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4756.golden.json index b72c1368c6d9..29b96fbe24d2 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4756.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4756.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4756", - "ingested": "2022-06-08T06:21:08.387537200Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4757.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4757.golden.json index d91aeda784d8..0193719b9a29 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4757.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4757.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4757", - "ingested": "2022-06-08T06:21:08.392196700Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4758.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4758.golden.json index 74efb603c38c..b27f572f3c2d 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4758.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4758.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4758", - "ingested": "2022-06-08T06:21:08.397524900Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4764.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4764.golden.json index c7bc4ea695b0..ab0e757d0414 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4764.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4764.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4764", - "ingested": "2022-06-08T06:21:08.402264500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4767_Account_Unlocked.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4767_Account_Unlocked.golden.json index 580a61a6c5dc..a194a3ff534a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4767_Account_Unlocked.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4767_Account_Unlocked.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4767", - "ingested": "2022-06-08T06:21:08.406967900Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4781_Account_Renamed.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4781_Account_Renamed.golden.json index 9a35ec91ecb1..0010cc0078c6 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4781_Account_Renamed.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4781_Account_Renamed.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4781", - "ingested": "2022-06-08T06:21:08.411904100Z", "kind": "event", "module": "security", "outcome": "success", @@ -92,7 +91,6 @@ "iam" ], "code": "4781", - "ingested": "2022-06-08T06:21:08.411917600Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4798.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4798.golden.json index 509e419a4068..460c9d3a84f2 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4798.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4798.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4798", - "ingested": "2022-06-08T06:21:08.418048300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4799.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4799.golden.json index d409e37cd1c9..6473c013f42a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4799.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4799.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4799", - "ingested": "2022-06-08T06:21:08.422696500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4964.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4964.golden.json index 43f17a4f460e..e62ac7e2cd74 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4964.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2016_4964.golden.json @@ -10,7 +10,6 @@ "iam" ], "code": "4964", - "ingested": "2022-06-08T06:21:08.427966400Z", "kind": "event", "module": "security", "outcome": "success", @@ -86,7 +85,6 @@ "iam" ], "code": "4964", - "ingested": "2022-06-08T06:21:08.427985300Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4688_Process_Created.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4688_Process_Created.golden.json index b25a579320a5..6ad492a4179a 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4688_Process_Created.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4688_Process_Created.golden.json @@ -10,7 +10,6 @@ "process" ], "code": "4688", - "ingested": "2022-06-08T06:21:08.435053500Z", "kind": "event", "module": "security", "outcome": "success", diff --git a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4689_Process_Exited.golden.json b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4689_Process_Exited.golden.json index b645ea0b1cf9..5ed8e0d2dde9 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4689_Process_Exited.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/ingest/security-windows2019_4689_Process_Exited.golden.json @@ -10,7 +10,6 @@ "process" ], "code": "4689", - "ingested": "2022-06-08T06:21:08.450696200Z", "kind": "event", "module": "security", "outcome": "success", @@ -83,7 +82,6 @@ "process" ], "code": "4689", - "ingested": "2022-06-08T06:21:08.450710900Z", "kind": "event", "module": "security", "outcome": "success", @@ -156,7 +154,6 @@ "process" ], "code": "4689", - "ingested": "2022-06-08T06:21:08.450714900Z", "kind": "event", "module": "security", "outcome": "success", From 41ab08cd6aa9e2c367b0c4b670f05f786a2862db Mon Sep 17 00:00:00 2001 From: Michael Katsoulis Date: Wed, 1 Nov 2023 15:58:01 +0200 Subject: [PATCH 49/50] Use filestream input as default for hints autodiscover. (#36950) * Use filestream input as default for hints autodiscover. Map co.elastic.logs/json* in hints to the ndjson parser of filestream * Update filebeat-kubernetes.yaml * Map co.elastic.logs/multiline.* hints to multiline parser of filestream input * Update documentation * Use file_identity.fingerprint as default way of file unique id creation --------- Co-authored-by: Andrew Gizas --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/filebeat-kubernetes.yaml | 23 +- .../filebeat/filebeat-configmap.yaml | 23 +- filebeat/autodiscover/builder/hints/config.go | 28 +- filebeat/autodiscover/builder/hints/logs.go | 52 +- .../autodiscover/builder/hints/logs_test.go | 444 +++++++++++++++--- filebeat/docs/autodiscover-hints.asciidoc | 31 +- .../input-filestream-reader-options.asciidoc | 1 + filebeat/harvester/util.go | 13 +- 9 files changed, 525 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3fc9493640b3..7d84cb5885f6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -254,6 +254,7 @@ is collected by it. - Avoid unwanted publication of Azure entity records. {pull}36753[36753] - Avoid unwanted publication of Okta entity records. {pull}36770[36770] - Add support for Digest Authentication to CEL input. {issue}35514[35514] {pull}36932[36932] +- Use filestream input with file_identity.fingerprint as default for hints autodiscover. {issue}35984[35984] {pull}36950[36950] *Auditbeat* diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index d1f8fd975e0f..497010cef90c 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -112,9 +112,16 @@ metadata: data: filebeat.yml: |- filebeat.inputs: - - type: container + - type: filestream paths: - /var/log/containers/*.log + parsers: + - container: ~ + prospector: + scanner: + fingerprint.enabled: true + symlinks: true + file_identity.fingerprint: ~ processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -123,15 +130,23 @@ data: logs_path: "/var/log/containers/" # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: - #filebeat.autodiscover: + # filebeat.autodiscover: # providers: # - type: kubernetes # node: ${NODE_NAME} # hints.enabled: true # hints.default_config: - # type: container + # type: filestream + # id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} # paths: - # - /var/log/containers/*${data.kubernetes.container.id}.log + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # parsers: + # - container: ~ + # prospector: + # scanner: + # fingerprint.enabled: true + # symlinks: true + # file_identity.fingerprint: ~ processors: - add_cloud_metadata: diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml index 6cbd6ad8b1a6..f2614e8c035b 100644 --- a/deploy/kubernetes/filebeat/filebeat-configmap.yaml +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -8,9 +8,16 @@ metadata: data: filebeat.yml: |- filebeat.inputs: - - type: container + - type: filestream paths: - /var/log/containers/*.log + parsers: + - container: ~ + prospector: + scanner: + fingerprint.enabled: true + symlinks: true + file_identity.fingerprint: ~ processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -19,15 +26,23 @@ data: logs_path: "/var/log/containers/" # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: - #filebeat.autodiscover: + # filebeat.autodiscover: # providers: # - type: kubernetes # node: ${NODE_NAME} # hints.enabled: true # hints.default_config: - # type: container + # type: filestream + # id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} # paths: - # - /var/log/containers/*${data.kubernetes.container.id}.log + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # parsers: + # - container: ~ + # prospector: + # scanner: + # fingerprint.enabled: true + # symlinks: true + # file_identity.fingerprint: ~ processors: - add_cloud_metadata: diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index dd1c2f759004..1161eb30e3f3 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -17,7 +17,9 @@ package hints -import conf "github.com/elastic/elastic-agent-libs/config" +import ( + conf "github.com/elastic/elastic-agent-libs/config" +) type config struct { Key string `config:"key"` @@ -26,11 +28,25 @@ type config struct { func defaultConfig() config { defaultCfgRaw := map[string]interface{}{ - "type": "container", + "type": "filestream", + "id": "kubernetes-container-logs-${data.kubernetes.container.id}", + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "fingerprint.enabled": true, + "symlinks": true, + }, + }, + "file_identity.fingerprint": nil, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "stream": "all", + "format": "auto", + }, + }, + }, "paths": []string{ - // To be able to use this builder with CRI-O replace paths with: - // /var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log - "/var/lib/docker/containers/${data.container.id}/*-json.log", + "/var/log/containers/*-${data.kubernetes.container.id}.log", }, } defaultCfg, _ := conf.NewConfigFrom(defaultCfgRaw) @@ -55,7 +71,7 @@ func (c *config) Unpack(from *conf.C) error { if len(fields) == 1 && fields[0] == "enabled" { // only enabling/disabling default config: if err := c.DefaultConfig.Merge(config); err != nil { - return nil + return err } } else { // full config provided, discard default. It must be a clone of the diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 52f8c91672a0..c39bfd533530 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -51,6 +51,8 @@ const ( processors = "processors" json = "json" pipeline = "pipeline" + ndjson = "ndjson" + parsers = "parsers" ) // validModuleNames to sanitize user input @@ -115,10 +117,20 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf continue } + inputType, _ := config.String("type", -1) tempCfg := mapstr.M{} - mline := l.getMultiline(h) - if len(mline) != 0 { - kubernetes.ShouldPut(tempCfg, multiline, mline, l.log) + + if mline := l.getMultiline(h); len(mline) != 0 { + if inputType == harvester.FilestreamType { + // multiline options should be under multiline parser in filestream input + parsersTempCfg := []mapstr.M{} + mlineTempCfg := mapstr.M{} + kubernetes.ShouldPut(mlineTempCfg, multiline, mline, l.log) + parsersTempCfg = append(parsersTempCfg, mlineTempCfg) + kubernetes.ShouldPut(tempCfg, parsers, parsersTempCfg, l.log) + } else { + kubernetes.ShouldPut(tempCfg, multiline, mline, l.log) + } } if ilines := l.getIncludeLines(h); len(ilines) != 0 { kubernetes.ShouldPut(tempCfg, includeLines, ilines, l.log) @@ -136,15 +148,24 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf } if jsonOpts := l.getJSONOptions(h); len(jsonOpts) != 0 { - kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log) + if inputType == harvester.FilestreamType { + // json options should be under ndjson parser in filestream input + parsersTempCfg := []mapstr.M{} + ndjsonTempCfg := mapstr.M{} + kubernetes.ShouldPut(ndjsonTempCfg, ndjson, jsonOpts, l.log) + parsersTempCfg = append(parsersTempCfg, ndjsonTempCfg) + kubernetes.ShouldPut(tempCfg, parsers, parsersTempCfg, l.log) + } else { + kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log) + } + } // Merge config template with the configs from the annotations // AppendValues option is used to append arrays from annotations to existing arrays while merging if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil { - logp.Debug("hints.builder", "config merge failed with error: %v", err) + l.log.Debugf("hints.builder", "config merge failed with error: %v", err) continue } - module := l.getModule(hints) if module != "" { moduleConf := map[string]interface{}{ @@ -154,9 +175,17 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf filesets := l.getFilesets(hints, module) for fileset, cfg := range filesets { filesetConf, _ := conf.NewConfigFrom(config) - - if inputType, _ := filesetConf.String("type", -1); inputType == harvester.ContainerType { + if inputType == harvester.ContainerType { _ = filesetConf.SetString("stream", -1, cfg.Stream) + } else if inputType == harvester.FilestreamType { + filestreamContainerParser := map[string]interface{}{ + "container": map[string]interface{}{ + "stream": cfg.Stream, + "format": "auto", + }, + } + parserCfg, _ := conf.NewConfigFrom(filestreamContainerParser) + _ = filesetConf.SetChild("parsers", 0, parserCfg) } else { _ = filesetConf.SetString("containers.stream", -1, cfg.Stream) } @@ -164,14 +193,13 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf moduleConf[fileset+".enabled"] = cfg.Enabled moduleConf[fileset+".input"] = filesetConf - logp.Debug("hints.builder", "generated config %+v", moduleConf) + l.log.Debugf("hints.builder", "generated config %+v", moduleConf) } config, _ = conf.NewConfigFrom(moduleConf) } - logp.Debug("hints.builder", "generated config %+v", config) + l.log.Debugf("hints.builder", "generated config %+v of logHints %+v", config, l) configs = append(configs, config) } - // Apply information in event to the template to generate the final config return template.ApplyConfigTemplate(event, configs) } @@ -222,7 +250,7 @@ func (l *logHints) getFilesets(hints mapstr.M, module string) map[string]*filese moduleFilesets, err := l.registry.ModuleAvailableFilesets(module) if err != nil { - logp.Err("Error retrieving module filesets: %+v", err) + l.log.Errorf("Error retrieving module filesets: %+v", err) return nil } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index cd2b72367712..4dc889e44d73 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -31,7 +31,7 @@ import ( ) func TestGenerateHints(t *testing.T) { - customCfg := conf.MustNewConfigFrom(map[string]interface{}{ + customDockerCfg := conf.MustNewConfigFrom(map[string]interface{}{ "default_config": map[string]interface{}{ "type": "docker", "containers": map[string]interface{}{ @@ -43,7 +43,7 @@ func TestGenerateHints(t *testing.T) { }, }) - customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{ + customContainerCfg := conf.MustNewConfigFrom(map[string]interface{}{ "default_config": map[string]interface{}{ "type": "container", "paths": []string{ @@ -61,6 +61,31 @@ func TestGenerateHints(t *testing.T) { }, }) + customFilestreamCfg := conf.MustNewConfigFrom(map[string]interface{}{ + "default_config": map[string]interface{}{ + "type": "filestream", + "id": "kubernetes-container-logs-${data.kubernetes.container.id}", + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "fingerprint.enabled": true, + "symlinks": true, + }, + }, + "file_identity.fingerprint": nil, + "paths": []string{ + "/var/log/containers/*-${data.kubernetes.container.id}.log", + }, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "stream": "all", + "format": "auto", + }, + }, + }, + }, + }) + defaultCfg := conf.NewConfig() defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{ @@ -77,7 +102,7 @@ func TestGenerateHints(t *testing.T) { result []mapstr.M }{ { - msg: "Default config is correct", + msg: "Default config is correct(default input)", config: defaultCfg, event: bus.Event{ "host": "1.2.3.4", @@ -95,8 +120,28 @@ func TestGenerateHints(t *testing.T) { len: 1, result: []mapstr.M{ { - "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, - "type": "container", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, }, @@ -120,7 +165,7 @@ func TestGenerateHints(t *testing.T) { result: []mapstr.M{}, }, { - msg: "Hint to enable when disabled by default works", + msg: "Hint to enable when disabled by default works(filestream)", config: defaultDisabled, event: bus.Event{ "host": "1.2.3.4", @@ -144,15 +189,35 @@ func TestGenerateHints(t *testing.T) { len: 1, result: []mapstr.M{ { - "type": "container", - "paths": []interface{}{"/var/lib/docker/containers/abc/*-json.log"}, + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, "exclude_lines": []interface{}{"^test2", "^test3"}, + "type": "filestream", }, }, }, { msg: "Hints without host should return nothing", - config: customCfg, + config: customDockerCfg, event: bus.Event{ "hints": mapstr.M{ "metrics": mapstr.M{ @@ -165,7 +230,7 @@ func TestGenerateHints(t *testing.T) { }, { msg: "Hints with logs.disable should return nothing", - config: customCfg, + config: customDockerCfg, event: bus.Event{ "hints": mapstr.M{ "logs": mapstr.M{ @@ -177,8 +242,8 @@ func TestGenerateHints(t *testing.T) { result: []mapstr.M{}, }, { - msg: "Empty event hints should return default config", - config: customCfg, + msg: "Empty event hints should return default config(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -204,8 +269,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with include|exclude_lines must be part of the input config", - config: customCfg, + msg: "Hint with include|exclude_lines must be part of the input config(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -239,8 +304,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hints with two sets of include|exclude_lines must be part of the input config", - config: customCfg, + msg: "Hints with two sets of include|exclude_lines must be part of the input config(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -285,8 +350,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with multiline config must have a multiline in the input config", - config: customCfg, + msg: "Hint with multiline config must have a multiline in the input config(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -324,8 +389,171 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with inputs config as json must be accepted", - config: customCfg, + msg: "Hint with multiline config must have a multiline in the input config parsers(filestream input)", + config: customFilestreamCfg, + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": mapstr.M{ + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + }, + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + "hints": mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + "negate": "true", + }, + }, + }, + }, + len: 1, + result: []mapstr.M{ + { + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, + map[string]interface{}{ + "multiline": map[string]interface{}{ + "pattern": "^test", + "negate": "true", + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", + }, + }, + }, + { + msg: "Hint with json config options must include them in the input config ndjson parser(filestream input)", + config: customFilestreamCfg, + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": mapstr.M{ + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + }, + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + "hints": mapstr.M{ + "logs": mapstr.M{ + "json": mapstr.M{ + "add_error_key": true, + "expand_keys": true, + }, + }, + }, + }, + len: 1, + result: []mapstr.M{ + { + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, + map[string]interface{}{ + "ndjson": map[string]interface{}{ + "add_error_key": true, + "expand_keys": true, + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", + }, + }, + }, + { + msg: "Hint with json config options must include them in the input config(container input)", + config: customContainerCfg, + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": mapstr.M{ + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + }, + "container": mapstr.M{ + "name": "foobar", + "id": "abc", + }, + "hints": mapstr.M{ + "logs": mapstr.M{ + "json": mapstr.M{ + "add_error_key": true, + "expand_keys": true, + }, + }, + }, + }, + len: 1, + result: []mapstr.M{ + { + "type": "container", + "paths": []interface{}{ + "/var/lib/docker/containers/abc/*-json.log", + }, + "close_timeout": "true", + "json": map[string]interface{}{ + "add_error_key": true, + "expand_keys": true, + }, + "processors": []interface{}{ + map[string]interface{}{ + "add_tags": map[string]interface{}{ + "tags": []interface{}{"web"}, + "target": "environment", + }, + }, + }, + }, + }, + }, + { + msg: "Hint with inputs config as json must be accepted(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -359,8 +587,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with processors config must have a processors in the input config", - config: customCfg, + msg: "Hint with processors config must have a processors in the input config(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -408,8 +636,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Processors in hints must be appended in the processors of the default config", - config: customProcessorCfg, + msg: "Processors in hints must be appended in the processors of the default config(container input)", + config: customContainerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -463,8 +691,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with module should attach input to its filesets", - config: customCfg, + msg: "Hint with module should attach input to its filesets(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -513,8 +741,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with module should honor defined filesets", - config: customCfg, + msg: "Hint with module should honor defined filesets(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -564,8 +792,8 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with module should honor defined filesets with streams", - config: customCfg, + msg: "Hint with module should honor defined filesets with streams(docker input)", + config: customDockerCfg, event: bus.Event{ "host": "1.2.3.4", "kubernetes": mapstr.M{ @@ -616,7 +844,7 @@ func TestGenerateHints(t *testing.T) { }, }, { - msg: "Hint with module should attach input to its filesets", + msg: "Hint with module should attach input to its filesets(default input)", config: defaultCfg, event: bus.Event{ "host": "1.2.3.4", @@ -643,28 +871,62 @@ func TestGenerateHints(t *testing.T) { "error": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ - "type": "container", - "stream": "all", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, "access": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ - "type": "container", - "stream": "all", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, }, }, }, { - msg: "Hint with module should honor defined filesets", + msg: "Hint with module should honor defined filesets(default input)", config: defaultCfg, event: bus.Event{ "host": "1.2.3.4", @@ -692,28 +954,62 @@ func TestGenerateHints(t *testing.T) { "access": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ - "type": "container", - "stream": "all", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, "error": map[string]interface{}{ "enabled": false, "input": map[string]interface{}{ - "type": "container", - "stream": "all", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "all", + }, + }, }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, }, }, }, { - msg: "Hint with module should honor defined filesets with streams", + msg: "Hint with module should honor defined filesets with streams(default input)", config: defaultCfg, event: bus.Event{ "host": "1.2.3.4", @@ -742,21 +1038,55 @@ func TestGenerateHints(t *testing.T) { "access": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ - "type": "container", - "stream": "stdout", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "stdout", + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, + }, + "type": "filestream", }, }, "error": map[string]interface{}{ "enabled": true, "input": map[string]interface{}{ - "type": "container", - "stream": "stderr", - "paths": []interface{}{ - "/var/lib/docker/containers/abc/*-json.log", + "id": "kubernetes-container-logs-abc", + "paths": []interface{}{"/var/log/containers/*-abc.log"}, + "parsers": []interface{}{ + map[string]interface{}{ + "container": map[string]interface{}{ + "format": "auto", + "stream": "stderr", + }, + }, + }, + "prospector": map[string]interface{}{ + "scanner": map[string]interface{}{ + "symlinks": true, + "fingerprint": map[string]interface{}{ + "enabled": true, + }, + }, + }, + "file_identity": map[string]interface{}{ + "fingerprint": nil, }, + "type": "filestream", }, }, }, diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc index d4fa08aee3d9..8a21edb8b2b6 100644 --- a/filebeat/docs/autodiscover-hints.asciidoc +++ b/filebeat/docs/autodiscover-hints.asciidoc @@ -2,7 +2,7 @@ hints in Kubernetes Pod annotations or Docker labels that have the prefix `co.elastic.logs`. As soon as the container starts, {beatname_uc} will check if it contains any hints and launch the proper config for it. Hints tell {beatname_uc} how to get logs for the given container. By default logs will be retrieved -from the container using the `container` input. You can use hints to modify this behavior. This is the full +from the container using the `filestream` input. You can use hints to modify this behavior. This is the full list of supported hints: [float] @@ -23,7 +23,34 @@ Multiline settings. See <> for a full list of all supported [float] ===== `co.elastic.logs/json.*` -JSON settings. See <> for a full list of all supported options. +JSON settings. In case of `filestream` input(default) see <> for a full list of all supported options. +In case of `container` or `log` input see <> for a full list of all supported options. + +For example, the following hints with json options: +[source,yaml] +----- +co.elastic.logs/json.message_key: "log" +co.elastic.logs/json.add_error_key: "true" +----- +will lead to the following input configuration: + +* `filestream` +[source,yaml] +----- +parsers: + - ndjson: + message_key: "log" + add_error_key: "true" +----- + +* `log` +[source,yaml] +----- +json.message_key: "log" +json.add_error_key: "true" +----- + +NOTE: `keys_under_root` json option of `log` input is replaced with `target` option in filestream input. Read the documentation(<>) on how to use it correctly. [float] ===== `co.elastic.logs/include_lines` diff --git a/filebeat/docs/inputs/input-filestream-reader-options.asciidoc b/filebeat/docs/inputs/input-filestream-reader-options.asciidoc index 9a6e86c146de..59d9018cb9ca 100644 --- a/filebeat/docs/inputs/input-filestream-reader-options.asciidoc +++ b/filebeat/docs/inputs/input-filestream-reader-options.asciidoc @@ -182,6 +182,7 @@ multiple lines. See <> for more information about configuring multiline options. [float] +[id="{beatname_lc}-input-{type}-ndjson"] ===== `ndjson` These options make it possible for {beatname_uc} to decode logs structured as diff --git a/filebeat/harvester/util.go b/filebeat/harvester/util.go index f39a718573a4..acfa19108512 100644 --- a/filebeat/harvester/util.go +++ b/filebeat/harvester/util.go @@ -21,12 +21,13 @@ import "github.com/elastic/beats/v7/libbeat/common/match" // Contains available input types const ( - LogType = "log" - StdinType = "stdin" - RedisType = "redis" - UdpType = "udp" - DockerType = "docker" - ContainerType = "container" + LogType = "log" + StdinType = "stdin" + RedisType = "redis" + UdpType = "udp" + DockerType = "docker" + ContainerType = "container" + FilestreamType = "filestream" ) // MatchAny checks if the text matches any of the regular expressions From 57df903385a9bdbb1d7bb2ad35e30b4445da206d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 1 Nov 2023 10:53:42 -0400 Subject: [PATCH 50/50] Update queue / ES connection defaults (#36990) --- CHANGELOG.next.asciidoc | 7 ++++++- libbeat/cmd/instance/beat_test.go | 4 ++-- libbeat/docs/queueconfig.asciidoc | 6 +++--- libbeat/outputs/elasticsearch/config.go | 12 ++++++++++-- .../elasticsearch/docs/elasticsearch.asciidoc | 4 ++-- libbeat/publisher/queue/memqueue/config.go | 7 ++++--- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7d84cb5885f6..35ff7048e39a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -11,7 +11,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* - The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] - Elastic-agent-autodiscover library updated to version 0.6.4, disabling metadata for deployment and cronjob. Pods that will be created from deployments or cronjobs will not have the extra metadata field for kubernetes.deployment or kubernetes.cronjob, respectively. {pull}36877[36877] - +- Defaults are changing for some options in the queue and Elasticsearch output, to improve typical performance based on current benchmark data. All changes can be overridden by setting them explicitly in the beat configuration. {pull}36990[36990] The changes are: + - `queue.mem.events` is changing from `4096` to `3200`. + - `queue.mem.flush.min_events` is changing from `2048` to `1600`. + - `queue.mem.flush.timeout` is changing from `1s` to `10s`. + - `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. + - `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. *Auditbeat* diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index 03474ecfcd91..52e55941225d 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -279,7 +279,7 @@ func TestPromoteOutputQueueSettings(t *testing.T) { }{ "blank": { input: []byte(""), - memEvents: 4096, + memEvents: 3200, }, "defaults": { input: []byte(` @@ -289,7 +289,7 @@ output: hosts: - "localhost:9200" `), - memEvents: 4096, + memEvents: 3200, }, "topLevelQueue": { input: []byte(` diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc index ade3bd2ec8ed..8fd4bed0416b 100644 --- a/libbeat/docs/queueconfig.asciidoc +++ b/libbeat/docs/queueconfig.asciidoc @@ -61,7 +61,7 @@ You can specify the following options in the `queue.mem` section of the +{beatna Number of events the queue can store. -The default value is 4096 events. +The default value is 3200 events. [float] ===== `flush.min_events` @@ -70,7 +70,7 @@ Minimum number of events required for publishing. If this value is set to 0, the output can start publishing events without additional waiting times. Otherwise the output has to wait for more events to become available. -The default value is 2048. +The default value is 1600. [float] ===== `flush.timeout` @@ -78,7 +78,7 @@ The default value is 2048. Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events will be immediately available for consumption. -The default value is 1s. +The default value is 10s. [float] [[configuration-internal-queue-disk]] diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index e504f2dc213c..6d8016b9636e 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -54,7 +54,7 @@ type Backoff struct { } const ( - defaultBulkSize = 50 + defaultBulkSize = 1600 ) var ( @@ -74,10 +74,18 @@ var ( Init: 1 * time.Second, Max: 60 * time.Second, }, - Transport: httpcommon.DefaultHTTPTransportSettings(), + Transport: esDefaultTransportSettings(), } ) +func esDefaultTransportSettings() httpcommon.HTTPTransportSettings { + transport := httpcommon.DefaultHTTPTransportSettings() + // The ES output differs from the common transport settings by having + // a 3-second idle timeout + transport.IdleConnTimeout = 3 * time.Second + return transport +} + func (c *elasticsearchConfig) Validate() error { if c.APIKey != "" && (c.Username != "" || c.Password != "") { return fmt.Errorf("cannot set both api_key and username/password") diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 6af56ac42db8..007bf92355dc 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -661,7 +661,7 @@ endif::[] ===== `bulk_max_size` -The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 50. +The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 1600. Events can be collected into batches. {beatname_uc} will split batches larger than `bulk_max_size` into multiple batches. @@ -693,7 +693,7 @@ Elasticsearch after a network error. The default is `60s`. The maximum amount of time an idle connection will remain idle before closing itself. Zero means no limit. The format is a Go language duration (example 60s is 60 seconds). -The default is 0. +The default is 3s. ===== `timeout` diff --git a/libbeat/publisher/queue/memqueue/config.go b/libbeat/publisher/queue/memqueue/config.go index 9e3be0a7611d..5e4f78ae41cc 100644 --- a/libbeat/publisher/queue/memqueue/config.go +++ b/libbeat/publisher/queue/memqueue/config.go @@ -32,9 +32,9 @@ type config struct { } var defaultConfig = config{ - Events: 4 * 1024, - FlushMinEvents: 2 * 1024, - FlushTimeout: 1 * time.Second, + Events: 3200, + FlushMinEvents: 1600, + FlushTimeout: 10 * time.Second, } func (c *config) Validate() error { @@ -53,6 +53,7 @@ func SettingsForUserConfig(cfg *c.C) (Settings, error) { return Settings{}, fmt.Errorf("couldn't unpack memory queue config: %w", err) } } + //nolint:gosimple // Actually want this conversion to be explicit since the types aren't definitionally equal. return Settings{ Events: config.Events, FlushMinEvents: config.FlushMinEvents,