From be200ae1f5113405ac8120dc8d07e4ca187be759 Mon Sep 17 00:00:00 2001 From: Hakan Vardar Date: Sat, 3 Aug 2024 04:57:10 +0300 Subject: [PATCH 1/4] EditorConfig file type #145 (#157) * EditorConfig file type * Removed deprecated ParseBytes function * Update pkg/filetype/file_type.go --------- Co-authored-by: Clayton Kehoe <118750525+kehoecj@users.noreply.github.com> --- go.mod | 4 +++- go.sum | 5 +++++ pkg/filetype/file_type.go | 9 +++++++++ pkg/validator/editorconfig.go | 18 ++++++++++++++++++ pkg/validator/validator_test.go | 2 ++ test/fixtures/good.editorconfig | 1 + test/fixtures/subdir2/bad.editorconfig | 2 ++ 7 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 pkg/validator/editorconfig.go create mode 100644 test/fixtures/good.editorconfig create mode 100644 test/fixtures/subdir2/bad.editorconfig diff --git a/go.mod b/go.mod index 1a90e610..8d4165c0 100644 --- a/go.mod +++ b/go.mod @@ -19,13 +19,15 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/editorconfig/editorconfig-core-go/v2 v2.6.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/go-envparse v0.1.0 // indirect github.com/mattn/go-colorable v0.1.9 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/zclconf/go-cty v1.13.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/text v0.11.0 // indirect diff --git a/go.sum b/go.sum index 5984a6d1..d15544fc 100644 --- a/go.sum +++ b/go.sum @@ -7,12 +7,15 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmms github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/editorconfig/editorconfig-core-go/v2 v2.6.2 h1:dKG8sc7n321deIVRcQtwlMNoBEra7j0qQ8RwxO8RN0w= +github.com/editorconfig/editorconfig-core-go/v2 v2.6.2/go.mod h1:7dvD3GCm7eBw53xZ/lsiq72LqobdMg3ITbMBxnmJmqY= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gurkankaymak/hocon v1.2.18 h1:/COj3okWh58himiYO0R7PrPX+iE7PbuzTn2cEv7fPsw= github.com/gurkankaymak/hocon v1.2.18/go.mod h1:dQCfhnuDKlLqAZRGhFTd81HkAfMx7STHv0w2JkJ6iq4= github.com/hashicorp/go-envparse v0.1.0 h1:bE++6bhIsNCPLvgDZkYqo3nA+/PFI51pkrHdmPSDFPY= @@ -48,6 +51,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/filetype/file_type.go b/pkg/filetype/file_type.go index 24795bd9..97f4c7bc 100644 --- a/pkg/filetype/file_type.go +++ b/pkg/filetype/file_type.go @@ -103,6 +103,14 @@ var EnvFileType = FileType{ validator.EnvValidator{}, } +// Instance of the FileType object to +// represent an EDITORCONFIG file +var EditorConfigFileType = FileType{ + "editorconfig", + misc.ArrToMap("editorconfig"), + validator.EditorConfigValidator{}, +} + // An array of files types that are supported // by the validator var FileTypes = []FileType{ @@ -117,4 +125,5 @@ var FileTypes = []FileType{ CsvFileType, HoconFileType, EnvFileType, + EditorConfigFileType, } diff --git a/pkg/validator/editorconfig.go b/pkg/validator/editorconfig.go new file mode 100644 index 00000000..17818ce4 --- /dev/null +++ b/pkg/validator/editorconfig.go @@ -0,0 +1,18 @@ +package validator + +import ( + "bytes" + + "github.com/editorconfig/editorconfig-core-go/v2" +) + +type EditorConfigValidator struct{} + +// Validate implements the Validator interface by attempting to +// parse a byte array of an editorconfig file using editorconfig-core-go package +func (EditorConfigValidator) Validate(b []byte) (bool, error) { + if _, err := editorconfig.Parse(bytes.NewReader(b)); err != nil { + return false, err + } + return true, nil +} diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index ac07aca4..210af41d 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -66,6 +66,8 @@ var testData = []struct { {"invalidHocon", []byte(`test = [1, 2,, 3]`), false, HoconValidator{}}, {"validEnv", []byte("KEY=VALUE"), true, EnvValidator{}}, {"invalidEnv", []byte("=TEST"), false, EnvValidator{}}, + {"validEditorConfig", []byte("working = true"), true, EditorConfigValidator{}}, + {"invalidEditorConfig", []byte("[*.md\nworking=false"), false, EditorConfigValidator{}}, } func Test_ValidationInput(t *testing.T) { diff --git a/test/fixtures/good.editorconfig b/test/fixtures/good.editorconfig new file mode 100644 index 00000000..e38660f1 --- /dev/null +++ b/test/fixtures/good.editorconfig @@ -0,0 +1 @@ +working = true \ No newline at end of file diff --git a/test/fixtures/subdir2/bad.editorconfig b/test/fixtures/subdir2/bad.editorconfig new file mode 100644 index 00000000..0f3e9fe6 --- /dev/null +++ b/test/fixtures/subdir2/bad.editorconfig @@ -0,0 +1,2 @@ +[*.md +working = false From 2982c50021ad3e9e4db81c7e8943c23d101dd305 Mon Sep 17 00:00:00 2001 From: Clayton Kehoe <118750525+kehoecj@users.noreply.github.com> Date: Fri, 2 Aug 2024 20:59:53 -0500 Subject: [PATCH 2/4] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fafd19a4..e61481f6 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ ## Supported config files formats: * Apple PList XML * CSV +* EDITORCONFIG * ENV * HCL * HOCON From d9a4db459a7f7361c375955f47b278ca3a8f88b6 Mon Sep 17 00:00:00 2001 From: Clayton Kehoe <118750525+kehoecj@users.noreply.github.com> Date: Fri, 2 Aug 2024 21:00:14 -0500 Subject: [PATCH 3/4] Update index.md --- index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/index.md b/index.md index 338f738e..dc3e67dc 100644 --- a/index.md +++ b/index.md @@ -53,6 +53,7 @@ ## Supported config files formats: * Apple PList XML * CSV +* EDITORCONFIG * ENV * HCL * HOCON From e601c40e6e9267bf44891b5645c8a5c69787f7a0 Mon Sep 17 00:00:00 2001 From: SkobelkinYaroslav <120472314+SkobelkinYaroslav@users.noreply.github.com> Date: Sat, 3 Aug 2024 04:04:05 +0200 Subject: [PATCH 4/4] small optimization for fsfinder (#136) * changed []string to map[string]struct{} in FileSystemFinder struct fields because of optimization (arrays were used only to check the occurrence of an element) * Changed field Extensions type in FileType from []string to map[string]struct{} * added requested changes * small refactoring of findOne func * fixd the return value * get back to trimprefix func * cleanup * fsfinder optimization * i hope it fixed * removed unnecessary calls to filepath.Abs * Apply suggestions from code review --------- Co-authored-by: Yaroslav <=> Co-authored-by: Clayton Kehoe <118750525+kehoecj@users.noreply.github.com> --- pkg/finder/finder_test.go | 14 ++++++++-- pkg/finder/fsfinder.go | 56 +++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pkg/finder/finder_test.go b/pkg/finder/finder_test.go index 4b5b8242..18ae27d5 100644 --- a/pkg/finder/finder_test.go +++ b/pkg/finder/finder_test.go @@ -166,9 +166,7 @@ func Test_FileSystemFinderMultipleFinder(t *testing.T) { func Test_FileSystemFinderDuplicateFiles(t *testing.T) { fsFinder := FileSystemFinderInit( WithPathRoots( - "../../test/fixtures/subdir/good.json", "../../test/fixtures/subdir/", - "../../test/fixtures/subdir/../subdir/good.json", ), ) @@ -250,3 +248,15 @@ func Test_FileFinderBadPath(t *testing.T) { t.Errorf("Error should be thrown for bad path") } } + +func Benchmark_Finder(b *testing.B) { + fsFinder := FileSystemFinderInit( + WithPathRoots("../../test/fixtures/"), + ) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = fsFinder.Find() + } +} diff --git a/pkg/finder/fsfinder.go b/pkg/finder/fsfinder.go index 01a81f37..08bb94a3 100644 --- a/pkg/finder/fsfinder.go +++ b/pkg/finder/fsfinder.go @@ -57,12 +57,14 @@ func WithDepth(depthVal int) FSFinderOptions { func FileSystemFinderInit(opts ...FSFinderOptions) *FileSystemFinder { defaultExcludeDirs := make(map[string]struct{}) + defaultExcludeFileTypes := make(map[string]struct{}) defaultPathRoots := []string{"."} fsfinder := &FileSystemFinder{ - PathRoots: defaultPathRoots, - FileTypes: filetype.FileTypes, - ExcludeDirs: defaultExcludeDirs, + PathRoots: defaultPathRoots, + FileTypes: filetype.FileTypes, + ExcludeDirs: defaultExcludeDirs, + ExcludeFileTypes: defaultExcludeFileTypes, } for _, opt := range opts { @@ -79,21 +81,11 @@ func (fsf FileSystemFinder) Find() ([]FileMetadata, error) { seen := make(map[string]struct{}, 0) uniqueMatches := make([]FileMetadata, 0) for _, pathRoot := range fsf.PathRoots { - matches, err := fsf.findOne(pathRoot) + matches, err := fsf.findOne(pathRoot, seen) if err != nil { return nil, err } - for _, match := range matches { - absPath, err := filepath.Abs(match.Path) - if err != nil { - return nil, err - } - if _, ok := seen[absPath]; ok { - continue - } - uniqueMatches = append(uniqueMatches, match) - seen[absPath] = struct{}{} - } + uniqueMatches = append(uniqueMatches, matches...) } return uniqueMatches, nil } @@ -101,7 +93,7 @@ func (fsf FileSystemFinder) Find() ([]FileMetadata, error) { // findOne recursively walks through all subdirectories (excluding the excluded subdirectories) // and identifying if the file matches a type defined in the fileTypes array for a // single path and returns the file metadata. -func (fsf FileSystemFinder) findOne(pathRoot string) ([]FileMetadata, error) { +func (fsf FileSystemFinder) findOne(pathRoot string, seenMap map[string]struct{}) ([]FileMetadata, error) { var matchingFiles []FileMetadata // check that the path exists before walking it or the error returned @@ -124,28 +116,40 @@ func (fsf FileSystemFinder) findOne(pathRoot string) ([]FileMetadata, error) { } // determine if directory is in the excludeDirs list or if the depth is greater than the maxDepth - _, isExcluded := fsf.ExcludeDirs[dirEntry.Name()] - if dirEntry.IsDir() && ((fsf.Depth != nil && strings.Count(path, string(os.PathSeparator)) > maxDepth) || isExcluded) { - return filepath.SkipDir + if dirEntry.IsDir() { + _, isExcluded := fsf.ExcludeDirs[dirEntry.Name()] + if isExcluded || (fsf.Depth != nil && strings.Count(path, string(os.PathSeparator)) > maxDepth) { + return filepath.SkipDir + } } if !dirEntry.IsDir() { // filepath.Ext() returns the extension name with a dot so it // needs to be removed. - walkFileExtension := strings.TrimPrefix(filepath.Ext(path), ".") + extensionLowerCase := strings.ToLower(walkFileExtension) - if _, ok := fsf.ExcludeFileTypes[walkFileExtension]; ok { + if _, isExcluded := fsf.ExcludeFileTypes[extensionLowerCase]; isExcluded { return nil } - extensionLowerCase := strings.ToLower(walkFileExtension) + for _, fileType := range fsf.FileTypes { - if _, ok := fileType.Extensions[extensionLowerCase]; ok { - fileMetadata := FileMetadata{dirEntry.Name(), path, fileType} - matchingFiles = append(matchingFiles, fileMetadata) - break + if _, isMatched := fileType.Extensions[extensionLowerCase]; isMatched { + absPath, err := filepath.Abs(path) + if err != nil { + return err + } + + if _, seen := seenMap[absPath]; !seen { + fileMetadata := FileMetadata{dirEntry.Name(), absPath, fileType} + matchingFiles = append(matchingFiles, fileMetadata) + seenMap[absPath] = struct{}{} + } + + return nil } } + fsf.ExcludeFileTypes[extensionLowerCase] = struct{}{} } return nil