From f6bdfacabe217c5de583abdfd4bcf27ecd175f8e Mon Sep 17 00:00:00 2001 From: plastikfan Date: Thu, 11 Jan 2024 15:14:25 +0000 Subject: [PATCH] fix(proxy): use poly filter and ensure journal files excluded (#105) --- Taskfile.yml | 4 +-- go.mod | 4 +-- go.sum | 4 +++ src/app/command/bootstrap.go | 29 ++++------------- src/app/command/shrink-cmd.go | 20 ++++++------ src/app/command/shrink-cmd_test.go | 2 +- src/app/proxy/controller-sampler_test.go | 18 +++++------ src/app/proxy/enter-shrink.go | 41 +++++++++++++++++------- src/app/proxy/image-defs.go | 4 +-- src/app/proxy/path-finder.go | 4 +-- test/data/configuration/pixa-test.yml | 2 ++ 11 files changed, 70 insertions(+), 62 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 6bbcec0..3616f1b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -29,9 +29,9 @@ tasks: c: cmds: - - task: clean + - task: clean-dist - clean: + clean-dist: cmds: - cmd: rm -rf {{.DIST_DIR}}/* diff --git a/go.mod b/go.mod index a43dafd..79e1b5e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 github.com/samber/lo v1.39.0 - github.com/snivilised/extendio v0.5.3 + github.com/snivilised/extendio v0.5.4 github.com/snivilised/lorax v0.4.4 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 @@ -46,7 +46,7 @@ require ( github.com/natefinch/lumberjack v2.0.0+incompatible github.com/nicksnyder/go-i18n/v2 v2.3.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect - github.com/snivilised/cobrass v0.4.2 + github.com/snivilised/cobrass v0.4.3 github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index b8622ef..452db5a 100644 --- a/go.sum +++ b/go.sum @@ -100,10 +100,14 @@ github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/snivilised/cobrass v0.4.2 h1:V2LqFKZWYqdewSRPeLFqyKPfOr96ihs4+ixSnP/528U= github.com/snivilised/cobrass v0.4.2/go.mod h1:bweMGepAjflKLqlCLkGi+Jvdru6ttdOr3L75rKq2yA8= +github.com/snivilised/cobrass v0.4.3 h1:uDXEgPRhC4Bc/n5qCK1ItMp62UL7tcw5Vz52QebEtv8= +github.com/snivilised/cobrass v0.4.3/go.mod h1:zY/ECw09lrTDrH1ynTCiCsIYsDNEaDRj7mxOKj2BsFE= github.com/snivilised/extendio v0.5.2 h1:gpaA5hGM0DCyY1PHy51fnmAI76tzCOOo4rteyuWSTDI= github.com/snivilised/extendio v0.5.2/go.mod h1:Ctp0GkEwMJ51FmgJE9nLLv86VTWILzgILdTgAeQ+BME= github.com/snivilised/extendio v0.5.3 h1:zuVT1GM2JzqCrC2i6MilKBdFBqGVA0fmwldlgGmEO4Q= github.com/snivilised/extendio v0.5.3/go.mod h1:2oHYhcClC43GVmhiUkQITujOqCW/RRWR4vXXl2VIfMo= +github.com/snivilised/extendio v0.5.4 h1:znyzAH3ynQ3onE+lfeqWvMBas2olx1ZWjLT+dn+dYv4= +github.com/snivilised/extendio v0.5.4/go.mod h1:2oHYhcClC43GVmhiUkQITujOqCW/RRWR4vXXl2VIfMo= github.com/snivilised/lorax v0.4.2 h1:jPo+o4cP/BUYBkR6sg3oLybHx0j2R+0wMLnvOf/gh0g= github.com/snivilised/lorax v0.4.2/go.mod h1:vyhM905Fc4fzShAXPvykS8ZRnO6B85hd0emJgZE4iNY= github.com/snivilised/lorax v0.4.4 h1:fxuuew+88yUC9JkTq2iQhGx8tvECBzj2ugjssWlTI6A= diff --git a/src/app/command/bootstrap.go b/src/app/command/bootstrap.go index d439b41..ba8954f 100644 --- a/src/app/command/bootstrap.go +++ b/src/app/command/bootstrap.go @@ -4,8 +4,6 @@ import ( "fmt" "log/slog" "os" - "path/filepath" - "strings" "github.com/cubiest/jibberjabber" "github.com/natefinch/lumberjack" @@ -27,6 +25,11 @@ import ( "github.com/snivilised/pixa/src/i18n" ) +const ( + defaultLogFilename = "pixa.log" + perm = 0o766 +) + type LocaleDetector interface { Scan() language.Tag } @@ -250,26 +253,6 @@ func (b *Bootstrap) viper() { b.LoggingCFG, _ = b.OptionsInfo.Config.Readers.Logging.Read(b.OptionsInfo.Config.Viper) } -func (b *Bootstrap) ensure(p string) string { - var ( - directory, file string - ) - - if strings.HasSuffix(p, string(os.PathSeparator)) { - directory = p - file = "pixa.log" - } else { - directory, file = filepath.Split(p) - } - - if !b.Vfs.DirectoryExists(directory) { - const perm = 0o766 - _ = b.Vfs.MkdirAll(directory, os.FileMode(perm)) - } - - return filepath.Clean(filepath.Join(directory, file)) -} - func (b *Bootstrap) level(raw string) zapcore.LevelEnabler { if l, err := zapcore.ParseLevel(raw); err == nil { return l @@ -290,7 +273,7 @@ func (b *Bootstrap) logger() *slog.Logger { } logPath = utils.ResolvePath(logPath) - logPath = b.ensure(logPath) + logPath, _ = utils.EnsurePathAt(logPath, defaultLogFilename, perm, b.Vfs) ws := zapcore.AddSync(&lumberjack.Logger{ Filename: logPath, diff --git a/src/app/command/shrink-cmd.go b/src/app/command/shrink-cmd.go index 772ef24..4b7663e 100644 --- a/src/app/command/shrink-cmd.go +++ b/src/app/command/shrink-cmd.go @@ -51,8 +51,10 @@ var shrinkShortFlags = cobrass.KnownByCollection{ "cpu": "C", // family: worker-pool "now": "N", // family: worker-pool "dry-run": "D", // family: preview + "files": "F", // family: filter "files-gb": "G", // family: filter "files-rx": "X", // family: filter + "folders": "F", // family: filter "folders-gb": "Z", // family: filter "folders-rx": "Y", // family: filter "profile": "P", // family: profile @@ -65,7 +67,7 @@ func init() { const ( shrinkPsName = "shrink-ps" - filesFamName = "files-family" + polyFamName = "poly-family" ) func newShrinkFlagInfoWithShort[T any](usage string, defaultValue T) *assistant.FlagInfo { @@ -322,13 +324,13 @@ func (b *Bootstrap) buildShrinkCommand(container *assistant.CobraContainer) *cob maxQuality, ) - // family: files [--files-gb(G), --files-rx(X)] + // family: poly [--files(f), --files-rx(X), --folders-gb(Z), --folders-rx(Y)] // - // The files filter family is not required on the root, so we define it + // A file filter is not required on the root, so we define it // here instead. // - filesFam := assistant.NewParamSet[store.FilesFilterParameterSet](shrinkCommand) - filesFam.Native.BindAll(filesFam) + polyFam := assistant.NewParamSet[store.PolyFilterParameterSet](shrinkCommand) + polyFam.Native.BindAll(polyFam) paramSet.Native.KnownBy = thirdPartyFlags @@ -352,7 +354,7 @@ func (b *Bootstrap) buildShrinkCommand(container *assistant.CobraContainer) *cob // container.MustRegisterRootedCommand(shrinkCommand) container.MustRegisterParamSet(shrinkPsName, paramSet) - container.MustRegisterParamSet(filesFamName, filesFam) + container.MustRegisterParamSet(polyFamName, polyFam) // TODO: we might need to code this via an anonymous func, store the vfs on // the bootstrap, then access it from the func, instead of using @@ -369,8 +371,8 @@ func (b *Bootstrap) getShrinkInputs() *proxy.ShrinkCommandInputs { ParamSet: b.Container.MustGetParamSet( shrinkPsName, ).(*assistant.ParamSet[proxy.ShrinkParameterSet]), - FilesFam: b.Container.MustGetParamSet( - filesFamName, - ).(*assistant.ParamSet[store.FilesFilterParameterSet]), + PolyFam: b.Container.MustGetParamSet( + polyFamName, + ).(*assistant.ParamSet[store.PolyFilterParameterSet]), } } diff --git a/src/app/command/shrink-cmd_test.go b/src/app/command/shrink-cmd_test.go index 535b0c0..5e50f83 100644 --- a/src/app/command/shrink-cmd_test.go +++ b/src/app/command/shrink-cmd_test.go @@ -207,7 +207,7 @@ var _ = Describe("ShrinkCmd", Ordered, func() { message: "with long form glob filtering options options", args: []string{ // "--folders-gb", "A*", - "--files-gb", "*.jpg", + "--files", "*.jpg", }, }, }), diff --git a/src/app/proxy/controller-sampler_test.go b/src/app/proxy/controller-sampler_test.go index 138bb32..18a5991 100644 --- a/src/app/proxy/controller-sampler_test.go +++ b/src/app/proxy/controller-sampler_test.go @@ -169,7 +169,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--gaussian-blur", "0.51", "--interlace", "line", }, @@ -188,7 +188,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--profile", "adaptive", "--gaussian-blur", "0.51", "--interlace", "line", @@ -209,7 +209,7 @@ var _ = Describe("SamplerController", Ordered, func() { "--sample", "--last", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--profile", "adaptive", "--gaussian-blur", "0.51", "--interlace", "line", @@ -228,7 +228,7 @@ var _ = Describe("SamplerController", Ordered, func() { relative: BackyardWorldsPlanet9Scan01, args: []string{ "--sample", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--profile", "adaptive", }, expected: helpers.BackyardWorldsPlanet9Scan01First2, @@ -267,7 +267,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--scheme", "singleton", "--gaussian-blur", "0.51", "--interlace", "line", @@ -287,7 +287,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--gaussian-blur", "0.51", "--interlace", "line", }, @@ -307,7 +307,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--profile", "adaptive", "--gaussian-blur", "0.51", "--interlace", "line", @@ -328,7 +328,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--scheme", "singleton", "--gaussian-blur", "0.51", "--interlace", "line", @@ -349,7 +349,7 @@ var _ = Describe("SamplerController", Ordered, func() { args: []string{ "--sample", "--no-files", "4", - "--files-gb", "*Backyard-Worlds*", + "--files", "*Backyard-Worlds*", "--strip", "--interlace", "plane", "--quality", "85", diff --git a/src/app/proxy/enter-shrink.go b/src/app/proxy/enter-shrink.go index 4a6f095..d844871 100644 --- a/src/app/proxy/enter-shrink.go +++ b/src/app/proxy/enter-shrink.go @@ -39,12 +39,13 @@ func (e *ShrinkEntry) LookAheadOptionsFn(o *nav.TraverseOptions) { return err }, } - - defs := e.getFilterDefs() - o.Store.FilterDefs = defs } func (e *ShrinkEntry) getFilterDefs() *nav.FilterDefinitions { + // the filter we expect the user to provide does not include the file suffix, + // it only applies to the base name and we define the suffix part of the filter + // internally. + // var ( file, folder *nav.FilterDef defs *nav.FilterDefinitions @@ -52,18 +53,32 @@ func (e *ShrinkEntry) getFilterDefs() *nav.FilterDefinitions { pattern string ) + const ( + defaultGbSuffix = "*.jp*g" + defaultRxSuffix = "(?i).(jpe?g|png)$" + ) + + extensions := "jpg,jpeg,png" + switch { - case e.Inputs.FilesFam.Native.FilesGlob != "": - pattern = e.Inputs.FilesFam.Native.FilesGlob + case e.Inputs.PolyFam.Native.Files != "": + pattern = fmt.Sprintf("%v|%v", e.Inputs.PolyFam.Native.Files, extensions) + file = &nav.FilterDef{ - Type: nav.FilterTypeGlobEn, - Description: fmt.Sprintf("--files-gb(G): '%v'", pattern), + Type: nav.FilterTypeExtendedGlobEn, + Description: fmt.Sprintf("--files(F): '%v'", pattern), Pattern: pattern, Scope: nav.ScopeFileEn, } - case e.Inputs.FilesFam.Native.FilesRexEx != "": - pattern = e.Inputs.FilesFam.Native.FilesRexEx + case e.Inputs.PolyFam.Native.FilesRexEx != "": + // we make the regex non case specific and also use a dot ta match + // any character before the suffix. Perhaps we need extendio to define + // an extended regex filter that has similar suffix functionality to + // the extended glob + // + pattern = fmt.Sprintf("(?i).%v.*(jpe?g|png)$", e.Inputs.PolyFam.Native.FilesRexEx) + file = &nav.FilterDef{ Type: nav.FilterTypeRegexEn, Description: fmt.Sprintf("--files-rx(X): '%v'", pattern), @@ -72,10 +87,10 @@ func (e *ShrinkEntry) getFilterDefs() *nav.FilterDefinitions { } default: - pattern = "(?i).(jpe?g|png)$" + pattern = fmt.Sprintf("*|%v", extensions) file = &nav.FilterDef{ - Type: nav.FilterTypeRegexEn, - Description: fmt.Sprintf("--files-rx(X): '%v'", pattern), + Type: nav.FilterTypeExtendedGlobEn, + Description: fmt.Sprintf("default extended glob filter: '%v'", pattern), Pattern: pattern, Scope: nav.ScopeFileEn, } @@ -215,6 +230,8 @@ func (e *ShrinkEntry) ConfigureOptions(o *nav.TraverseOptions) { finder: finder, fileManager: e.FileManager, }) + + o.Store.FilterDefs = e.getFilterDefs() } func clearResumeFromWith(with nav.CreateNewRunnerWith) nav.CreateNewRunnerWith { diff --git a/src/app/proxy/image-defs.go b/src/app/proxy/image-defs.go index c313818..7aeae41 100644 --- a/src/app/proxy/image-defs.go +++ b/src/app/proxy/image-defs.go @@ -125,7 +125,7 @@ type RootCommandInputs struct { ParamSet *assistant.ParamSet[RootParameterSet] PreviewFam *assistant.ParamSet[store.PreviewParameterSet] WorkerPoolFam *assistant.ParamSet[store.WorkerPoolParameterSet] - FoldersFam *assistant.ParamSet[store.FoldersFilterParameterSet] + FoldersFam *assistant.ParamSet[store.FoldersFilterParameterSet] // !!! ProfileFam *assistant.ParamSet[store.ProfileParameterSet] CascadeFam *assistant.ParamSet[store.CascadeParameterSet] } @@ -133,5 +133,5 @@ type RootCommandInputs struct { type ShrinkCommandInputs struct { Root *RootCommandInputs ParamSet *assistant.ParamSet[ShrinkParameterSet] - FilesFam *assistant.ParamSet[store.FilesFilterParameterSet] + PolyFam *assistant.ParamSet[store.PolyFilterParameterSet] } diff --git a/src/app/proxy/path-finder.go b/src/app/proxy/path-finder.go index 0f4881c..2c5d720 100644 --- a/src/app/proxy/path-finder.go +++ b/src/app/proxy/path-finder.go @@ -231,9 +231,9 @@ type PathFinder struct { } func (f *PathFinder) JournalFile(item *nav.TraverseItem) string { - path := FilenameWithoutExtension(item.Extension.Name) + f.statics.journal + file := FilenameWithoutExtension(item.Extension.Name) + f.statics.journal - return filepath.Join(item.Extension.Parent, path) + return filepath.Join(item.Extension.Parent, file) } // Destination returns the location of what should be used diff --git a/test/data/configuration/pixa-test.yml b/test/data/configuration/pixa-test.yml index 5712b45..a1b502f 100644 --- a/test/data/configuration/pixa-test.yml +++ b/test/data/configuration/pixa-test.yml @@ -33,6 +33,8 @@ advanced: legacy: .LEGACY journal-suffix: .journal.txt trash: TRASH + extensions: + suffixes: "jpg,png" logging: log-path: "~/snivilised/pixa/pixa.log" max-size: 10