From 900137649bcc798fdbec9e9f7bbd60faab44c908 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:03:45 +0000 Subject: [PATCH] Fix the performance regression when ingesting files on Windows (#37301) (#37360) Adding additional file system metadata caused additional system calls that degraded the file reading performance. (cherry picked from commit 8f50ca3240816a2db796f83c02aca63ab2710591) Co-authored-by: Denis --- .../filestream/copytruncate_prospector.go | 3 +- filebeat/input/filestream/environment_test.go | 9 +- filebeat/input/filestream/fswatch.go | 9 +- filebeat/input/filestream/fswatch_test.go | 129 +++++++++--------- filebeat/input/filestream/identifier.go | 3 +- .../filestream/identifier_inode_deviceid.go | 3 +- filebeat/input/filestream/identifier_test.go | 4 +- .../internal/input-logfile/fswatch.go | 8 +- filebeat/input/filestream/logger.go | 8 +- filebeat/input/filestream/prospector_test.go | 31 +++-- libbeat/common/file/file_info.go | 53 +++++++ libbeat/common/file/file_other.go | 9 +- libbeat/common/file/file_windows.go | 17 ++- .../reader/readfile/fs_metafields_other.go | 5 +- .../reader/readfile/fs_metafields_windows.go | 5 +- libbeat/reader/readfile/metafields.go | 6 +- .../reader/readfile/metafields_other_test.go | 8 +- .../readfile/metafields_windows_test.go | 8 +- 18 files changed, 199 insertions(+), 119 deletions(-) create mode 100644 libbeat/common/file/file_info.go diff --git a/filebeat/input/filestream/copytruncate_prospector.go b/filebeat/input/filestream/copytruncate_prospector.go index 10884cb9b94e..5b1c6bdd4277 100644 --- a/filebeat/input/filestream/copytruncate_prospector.go +++ b/filebeat/input/filestream/copytruncate_prospector.go @@ -28,6 +28,7 @@ import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" ) @@ -330,7 +331,7 @@ func (p *copyTruncateFileProspector) onRotatedFile( return } descCopy := fe.Descriptor - descCopy.Info = fi + descCopy.Info = file.ExtendFileInfo(fi) originalSrc := p.identifier.GetSource(loginp.FSEvent{NewPath: originalPath, Descriptor: descCopy}) p.rotatedFiles.addOriginalFile(originalPath, originalSrc) p.rotatedFiles.addRotatedFile(originalPath, fe.NewPath, src) diff --git a/filebeat/input/filestream/environment_test.go b/filebeat/input/filestream/environment_test.go index 1713a964c527..7c3c8ccd4d3b 100644 --- a/filebeat/input/filestream/environment_test.go +++ b/filebeat/input/filestream/environment_test.go @@ -37,6 +37,7 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" @@ -372,7 +373,13 @@ func (e *inputTestingEnvironment) getRegistryState(key string) (registryEntry, e func getIDFromPath(filepath, inputID string, fi os.FileInfo) string { identifier, _ := newINodeDeviceIdentifier(nil) - src := identifier.GetSource(loginp.FSEvent{Descriptor: loginp.FileDescriptor{Info: fi}, Op: loginp.OpCreate, NewPath: filepath}) + src := identifier.GetSource(loginp.FSEvent{ + Descriptor: loginp.FileDescriptor{ + Info: file.ExtendFileInfo(fi), + }, + Op: loginp.OpCreate, + NewPath: filepath, + }) return "filestream::" + inputID + "::" + src.Name() } diff --git a/filebeat/input/filestream/fswatch.go b/filebeat/input/filestream/fswatch.go index a32b4409ef2d..454a5b428b05 100644 --- a/filebeat/input/filestream/fswatch.go +++ b/filebeat/input/filestream/fswatch.go @@ -32,6 +32,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input/file" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + commonfile "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/match" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -406,7 +407,7 @@ type ingestTarget struct { filename string originalFilename string symlink bool - info os.FileInfo + info commonfile.ExtendedFileInfo } func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err error) { @@ -421,10 +422,11 @@ func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err err it.filename = filename it.originalFilename = filename - it.info, err = os.Lstat(it.filename) // to determine if it's a symlink + info, err := os.Lstat(it.filename) // to determine if it's a symlink if err != nil { return it, fmt.Errorf("failed to lstat %q: %w", it.filename, err) } + it.info = commonfile.ExtendFileInfo(info) if it.info.IsDir() { return it, fmt.Errorf("file %q is a directory", it.filename) @@ -438,10 +440,11 @@ func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err err } // now we know it's a symlink, we stat with link resolution - it.info, err = os.Stat(it.filename) + info, err := os.Stat(it.filename) if err != nil { return it, fmt.Errorf("failed to stat the symlink %q: %w", it.filename, err) } + it.info = commonfile.ExtendFileInfo(info) it.originalFilename, err = filepath.EvalSymlinks(it.filename) if err != nil { diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go index f9f58734360c..6c9d88b858e2 100644 --- a/filebeat/input/filestream/fswatch_test.go +++ b/filebeat/input/filestream/fswatch_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -68,7 +69,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 5}, // 5 bytes written + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 5}), // 5 bytes written }, } requireEqualEvents(t, expEvent, e) @@ -91,7 +92,7 @@ scanner: Op: loginp.OpWrite, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 10}, // +5 bytes appended + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 10}), // +5 bytes appended }, } requireEqualEvents(t, expEvent, e) @@ -113,7 +114,7 @@ scanner: Op: loginp.OpRename, Descriptor: loginp.FileDescriptor{ Filename: newFilename, - Info: testFileInfo{name: newBasename, size: 10}, + Info: file.ExtendFileInfo(&testFileInfo{name: newBasename, size: 10}), }, } requireEqualEvents(t, expEvent, e) @@ -133,7 +134,7 @@ scanner: Op: loginp.OpTruncate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -153,7 +154,7 @@ scanner: Op: loginp.OpTruncate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -172,7 +173,7 @@ scanner: Op: loginp.OpDelete, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -210,7 +211,7 @@ scanner: Descriptor: loginp.FileDescriptor{ Filename: filename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -241,7 +242,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -298,7 +299,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 5}, // +5 bytes appended + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 5}), // +5 bytes appended }, } requireEqualEvents(t, expEvent, e) @@ -332,7 +333,7 @@ scanner: Descriptor: loginp.FileDescriptor{ Filename: filename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -384,7 +385,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: firstFilename, - Info: testFileInfo{name: firstBasename, size: 5}, // "line\n" + Info: file.ExtendFileInfo(&testFileInfo{name: firstBasename, size: 5}), // "line\n" }, }, { @@ -392,7 +393,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: secondFilename, - Info: testFileInfo{name: secondBasename, size: 5}, // "line\n" + Info: file.ExtendFileInfo(&testFileInfo{name: secondBasename, size: 5}), // "line\n" }, }, } @@ -494,38 +495,38 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -543,31 +544,31 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -586,24 +587,24 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -617,17 +618,17 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, }, }, @@ -646,10 +647,10 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -663,10 +664,10 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -680,17 +681,17 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -709,34 +710,34 @@ scanner: normalFilename: { Filename: normalFilename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, Fingerprint: "bd151321c3bbdb44185414a1b56b5649a00206dd4792e7230db8904e43987336", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, Fingerprint: "bfdb99a65297062658c26dfcea816d76065df2a2da2594bfd9b96e9e405da1c2", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, Fingerprint: "c4058942bffcea08810a072d5966dfa5c06eb79b902bf0011890dd8d22e1a5f8", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -755,35 +756,35 @@ scanner: normalFilename: { Filename: normalFilename, Fingerprint: "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, // undersizedFilename got excluded because of the matching fingerprint excludedFilename: { Filename: excludedFilename, Fingerprint: "9c225a1e6a7df9c869499e923565b93937e88382bb9188145f117195cd41dcd1", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, Fingerprint: "7985b2b9750bdd3c76903db408aff3859204d6334279eaf516ecaeb618a218d5", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, Fingerprint: "da437600754a8eed6c194b7241b078679551c06c7dc89685a9a71be7829ad7e5", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go index 0cfeb031d633..a0cd7903e7ac 100644 --- a/filebeat/input/filestream/identifier.go +++ b/filebeat/input/filestream/identifier.go @@ -21,7 +21,6 @@ import ( "fmt" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" ) @@ -114,7 +113,7 @@ func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource { oldPath: e.OldPath, truncated: e.Op == loginp.OpTruncate, archived: e.Op == loginp.OpArchived, - fileID: i.name + identitySep + file.GetOSState(e.Descriptor.Info).String(), + fileID: i.name + identitySep + e.Descriptor.Info.GetOSState().String(), identifierGenerator: i.name, } } diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go index af6a56100862..05a768d2babf 100644 --- a/filebeat/input/filestream/identifier_inode_deviceid.go +++ b/filebeat/input/filestream/identifier_inode_deviceid.go @@ -27,7 +27,6 @@ import ( "time" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -93,7 +92,7 @@ func (i *inodeMarkerIdentifier) markerContents() string { } func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource { - osstate := file.GetOSState(e.Descriptor.Info) + osstate := e.Descriptor.Info.GetOSState() return fileSource{ desc: e.Descriptor, newPath: e.NewPath, diff --git a/filebeat/input/filestream/identifier_test.go b/filebeat/input/filestream/identifier_test.go index ca67ba375d60..1fcd4d73efa2 100644 --- a/filebeat/input/filestream/identifier_test.go +++ b/filebeat/input/filestream/identifier_test.go @@ -53,7 +53,7 @@ func TestFileIdentifier(t *testing.T) { src := identifier.GetSource(loginp.FSEvent{ NewPath: tmpFile.Name(), - Descriptor: loginp.FileDescriptor{Info: fi}, + Descriptor: loginp.FileDescriptor{Info: file.ExtendFileInfo(fi)}, }) assert.Equal(t, identifier.Name()+"::"+file.GetOSState(fi).String(), src.Name()) @@ -77,7 +77,7 @@ func TestFileIdentifier(t *testing.T) { src := identifier.GetSource(loginp.FSEvent{ NewPath: tmpFile.Name(), - Descriptor: loginp.FileDescriptor{Info: fi}, + Descriptor: loginp.FileDescriptor{Info: file.ExtendFileInfo(fi)}, }) assert.Equal(t, identifier.Name()+"::"+file.GetOSState(fi).String()+"-my-suffix", src.Name()) diff --git a/filebeat/input/filestream/internal/input-logfile/fswatch.go b/filebeat/input/filestream/internal/input-logfile/fswatch.go index dc00519c437b..1f91515036fa 100644 --- a/filebeat/input/filestream/internal/input-logfile/fswatch.go +++ b/filebeat/input/filestream/internal/input-logfile/fswatch.go @@ -18,11 +18,9 @@ package input_logfile import ( - "os" - "github.com/elastic/go-concert/unison" - file_helper "github.com/elastic/beats/v7/libbeat/common/file" + "github.com/elastic/beats/v7/libbeat/common/file" ) const ( @@ -63,7 +61,7 @@ type FileDescriptor struct { // the filename from the `Info`. Filename string // Info is the result of file stat - Info os.FileInfo + Info file.ExtendedFileInfo // Fingerprint is a computed hash of the file header Fingerprint string } @@ -75,7 +73,7 @@ func (fd FileDescriptor) FileID() string { if fd.Fingerprint != "" { return fd.Fingerprint } - return file_helper.GetOSState(fd.Info).String() + return fd.Info.GetOSState().String() } // SameFile returns true if descriptors point to the same file. diff --git a/filebeat/input/filestream/logger.go b/filebeat/input/filestream/logger.go index 7b644fd0d877..62cb416cdb26 100644 --- a/filebeat/input/filestream/logger.go +++ b/filebeat/input/filestream/logger.go @@ -19,7 +19,6 @@ package filestream import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/logp" ) @@ -31,8 +30,11 @@ func loggerWithEvent(logger *logp.Logger, event loginp.FSEvent, src loginp.Sourc if event.Descriptor.Fingerprint != "" { log = log.With("fingerprint", event.Descriptor.Fingerprint) } - if event.Descriptor.Info != nil && event.Descriptor.Info.Sys() != nil { - log = log.With("os_id", file.GetOSState(event.Descriptor.Info)) + if event.Descriptor.Info != nil { + osID := event.Descriptor.Info.GetOSState().String() + if osID != "" { + log = log.With("os_id", osID) + } } if event.NewPath != "" { log = log.With("new_path", event.NewPath) diff --git a/filebeat/input/filestream/prospector_test.go b/filebeat/input/filestream/prospector_test.go index 834784c81da7..552b4218c784 100644 --- a/filebeat/input/filestream/prospector_test.go +++ b/filebeat/input/filestream/prospector_test.go @@ -32,6 +32,7 @@ import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" @@ -139,7 +140,7 @@ func TestProspector_InitUpdateIdentifiers(t *testing.T) { }, }, filesOnDisk: map[string]loginp.FileDescriptor{ - tmpFileName: {Info: fi}, + tmpFileName: {Info: file.ExtendFileInfo(fi)}, }, expectedUpdatedKeys: map[string]string{"not_path::key1": "path::" + tmpFileName}, }, @@ -205,12 +206,12 @@ func TestProspectorNewAndUpdatedFiles(t *testing.T) { { Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), }, { Op: loginp.OpWrite, NewPath: "/path/to/other/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), }, }, ignoreOlder: 10 * time.Second, @@ -223,12 +224,12 @@ func TestProspectorNewAndUpdatedFiles(t *testing.T) { { Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), }, { Op: loginp.OpWrite, NewPath: "/path/to/other/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), }, }, ignoreOlder: 5 * time.Minute, @@ -268,12 +269,12 @@ func TestProspectorHarvesterUpdateIgnoredFiles(t *testing.T) { eventCreate := loginp.FSEvent{ Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), } eventUpdated := loginp.FSEvent{ Op: loginp.OpWrite, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 10, time.Now(), nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 10, time.Now(), nil}), } expectedEvents := []harvesterEvent{ harvesterStart("path::/path/to/file"), @@ -730,20 +731,20 @@ type testFileInfo struct { sys interface{} } -func (t testFileInfo) Name() string { return t.name } -func (t testFileInfo) Size() int64 { return t.size } -func (t testFileInfo) Mode() os.FileMode { return 0 } -func (t testFileInfo) ModTime() time.Time { return t.time } -func (t testFileInfo) IsDir() bool { return false } -func (t testFileInfo) Sys() interface{} { return t.sys } +func (t *testFileInfo) Name() string { return t.name } +func (t *testFileInfo) Size() int64 { return t.size } +func (t *testFileInfo) Mode() os.FileMode { return 0 } +func (t *testFileInfo) ModTime() time.Time { return t.time } +func (t *testFileInfo) IsDir() bool { return false } +func (t *testFileInfo) Sys() interface{} { return t.sys } func createTestFileDescriptor() loginp.FileDescriptor { - return createTestFileDescriptorWithInfo(testFileInfo{}) + return createTestFileDescriptorWithInfo(&testFileInfo{}) } func createTestFileDescriptorWithInfo(fi fs.FileInfo) loginp.FileDescriptor { return loginp.FileDescriptor{ - Info: fi, + Info: file.ExtendFileInfo(fi), Fingerprint: "fingerprint", Filename: "filename", } diff --git a/libbeat/common/file/file_info.go b/libbeat/common/file/file_info.go new file mode 100644 index 000000000000..1364cf8b193d --- /dev/null +++ b/libbeat/common/file/file_info.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file + +import ( + "os" +) + +type ExtendedFileInfo interface { + os.FileInfo + GetOSState() StateOS +} + +type extendedFileInfo struct { + os.FileInfo + osSpecific *StateOS +} + +// GetOSState returns the platform specific StateOS. +// The data is fetched once and cached. +func (f *extendedFileInfo) GetOSState() StateOS { + if f == nil || f.FileInfo == nil { + return StateOS{} + } + + if f.osSpecific != nil { + return *f.osSpecific + } + + osSpecific := GetOSState(f.FileInfo) + f.osSpecific = &osSpecific + return osSpecific +} + +// ExtendFileInfo wraps the standard FileInfo with an extended version. +func ExtendFileInfo(fi os.FileInfo) ExtendedFileInfo { + return &extendedFileInfo{FileInfo: fi} +} diff --git a/libbeat/common/file/file_other.go b/libbeat/common/file/file_other.go index 600e225bc189..3ba429ced201 100644 --- a/libbeat/common/file/file_other.go +++ b/libbeat/common/file/file_other.go @@ -32,7 +32,14 @@ type StateOS struct { // GetOSState returns the FileStateOS for non windows systems func GetOSState(info os.FileInfo) StateOS { - stat := info.Sys().(*syscall.Stat_t) + sys := info.Sys() + if sys == nil { + return StateOS{} + } + stat, ok := sys.(*syscall.Stat_t) + if !ok { + return StateOS{} + } // Convert inode and dev to uint64 to be cross platform compatible fileState := StateOS{ diff --git a/libbeat/common/file/file_windows.go b/libbeat/common/file/file_windows.go index 1b8a9da49dea..3c541c942d2c 100644 --- a/libbeat/common/file/file_windows.go +++ b/libbeat/common/file/file_windows.go @@ -42,6 +42,9 @@ var ( // GetOSState returns the platform specific StateOS func GetOSState(info os.FileInfo) StateOS { + if info == nil { + return StateOS{} + } // os.SameFile must be called to populate the id fields. Otherwise in case for example // os.Stat(file) is used to get the fileInfo, the ids are empty. // https://github.com/elastic/beats/filebeat/pull/53 @@ -56,14 +59,22 @@ func GetOSState(info os.FileInfo) StateOS { // Uint should already return uint64, but making sure this is the case // The required fields can be found here: https://github.com/golang/go/blob/master/src/os/types_windows.go#L78 fileState := StateOS{ - IdxHi: uint64(fileStat.FieldByName("idxhi").Uint()), - IdxLo: uint64(fileStat.FieldByName("idxlo").Uint()), - Vol: uint64(fileStat.FieldByName("vol").Uint()), + IdxHi: getFieldValue(fileStat, "idxhi"), + IdxLo: getFieldValue(fileStat, "idxlo"), + Vol: getFieldValue(fileStat, "vol"), } return fileState } +func getFieldValue(val reflect.Value, name string) uint64 { + fieldValue := val.FieldByName(name) + if !fieldValue.IsValid() { + return 0 + } + return uint64(fieldValue.Uint()) +} + // IsSame file checks if the files are identical func (fs StateOS) IsSame(state StateOS) bool { return fs.IdxHi == state.IdxHi && fs.IdxLo == state.IdxLo && fs.Vol == state.Vol diff --git a/libbeat/reader/readfile/fs_metafields_other.go b/libbeat/reader/readfile/fs_metafields_other.go index cc764c4bbcc7..1c1bcb8a24b8 100644 --- a/libbeat/reader/readfile/fs_metafields_other.go +++ b/libbeat/reader/readfile/fs_metafields_other.go @@ -21,7 +21,6 @@ package readfile import ( "fmt" - "os" "strconv" "github.com/elastic/beats/v7/libbeat/common/file" @@ -33,8 +32,8 @@ const ( inodeKey = "log.file.inode" ) -func setFileSystemMetadata(fi os.FileInfo, fields mapstr.M) error { - osstate := file.GetOSState(fi) +func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M) error { + osstate := fi.GetOSState() _, err := fields.Put(deviceIDKey, strconv.FormatUint(osstate.Device, 10)) if err != nil { return fmt.Errorf("failed to set %q: %w", deviceIDKey, err) diff --git a/libbeat/reader/readfile/fs_metafields_windows.go b/libbeat/reader/readfile/fs_metafields_windows.go index 97bfd5c72de1..0a928ba10474 100644 --- a/libbeat/reader/readfile/fs_metafields_windows.go +++ b/libbeat/reader/readfile/fs_metafields_windows.go @@ -19,7 +19,6 @@ package readfile import ( "fmt" - "os" "strconv" "github.com/elastic/beats/v7/libbeat/common/file" @@ -32,8 +31,8 @@ const ( volKey = "log.file.vol" ) -func setFileSystemMetadata(fi os.FileInfo, fields mapstr.M) error { - osstate := file.GetOSState(fi) +func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M) error { + osstate := fi.GetOSState() _, err := fields.Put(idxhiKey, strconv.FormatUint(osstate.IdxHi, 10)) if err != nil { return fmt.Errorf("failed to set %q: %w", idxhiKey, err) diff --git a/libbeat/reader/readfile/metafields.go b/libbeat/reader/readfile/metafields.go index c4c41e980f67..be74a9e07a57 100644 --- a/libbeat/reader/readfile/metafields.go +++ b/libbeat/reader/readfile/metafields.go @@ -19,8 +19,8 @@ package readfile import ( "fmt" - "os" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/reader" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -30,14 +30,14 @@ import ( type FileMetaReader struct { reader reader.Reader path string - fi os.FileInfo + fi file.ExtendedFileInfo fingerprint string offset int64 } // New creates a new Encode reader from input reader by applying // the given codec. -func NewFilemeta(r reader.Reader, path string, fi os.FileInfo, fingerprint string, offset int64) reader.Reader { +func NewFilemeta(r reader.Reader, path string, fi file.ExtendedFileInfo, fingerprint string, offset int64) reader.Reader { return &FileMetaReader{r, path, fi, fingerprint, offset} } diff --git a/libbeat/reader/readfile/metafields_other_test.go b/libbeat/reader/readfile/metafields_other_test.go index b9d25b854204..351d175156f3 100644 --- a/libbeat/reader/readfile/metafields_other_test.go +++ b/libbeat/reader/readfile/metafields_other_test.go @@ -20,23 +20,23 @@ package readfile import ( - "os" "syscall" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/mapstr" ) -func createTestFileInfo() os.FileInfo { - return testFileInfo{ +func createTestFileInfo() file.ExtendedFileInfo { + return file.ExtendFileInfo(testFileInfo{ name: "filename", size: 42, time: time.Now(), sys: &syscall.Stat_t{Dev: 17, Ino: 999}, - } + }) } func checkFields(t *testing.T, expected, actual mapstr.M) { diff --git a/libbeat/reader/readfile/metafields_windows_test.go b/libbeat/reader/readfile/metafields_windows_test.go index dce0b8d2161a..fb4cf2a5160a 100644 --- a/libbeat/reader/readfile/metafields_windows_test.go +++ b/libbeat/reader/readfile/metafields_windows_test.go @@ -18,12 +18,12 @@ package readfile import ( - "os" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -34,8 +34,8 @@ type winTestInfo struct { vol uint32 } -func createTestFileInfo() os.FileInfo { - return &winTestInfo{ +func createTestFileInfo() file.ExtendedFileInfo { + return file.ExtendFileInfo(&winTestInfo{ testFileInfo: testFileInfo{ name: "filename", size: 42, @@ -44,7 +44,7 @@ func createTestFileInfo() os.FileInfo { idxhi: 100, idxlo: 200, vol: 300, - } + }) } func checkFields(t *testing.T, expected, actual mapstr.M) {