From f64a1bd0a7e41c46e160fa8c13d752db01e27739 Mon Sep 17 00:00:00 2001 From: Dalton Tan Date: Thu, 14 Oct 2021 22:42:37 +0800 Subject: [PATCH] batching for stdout --- README.md | 61 +++++++++++++++++++++++++++++++++----------- go.mod | 1 + go.sum | 2 ++ xnotify.go | 74 ++++++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 105 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 71b3e90..c31f3b5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ without relying on polling. ## Installation -Download the precompiled binaries at the [release page](https://github.com/AgentCosmic/xnotify/releases). +Download the pre-compiled binaries at the [release page](https://github.com/AgentCosmic/xnotify/releases). Or if you have Go installed you can run: ```go get github.com/AgentCosmic/xnotify``` @@ -19,21 +19,28 @@ Or if you have Go installed you can run: ## Tutorial ``` +NAME: + xnotify - Watch files for changes. + File changes will be printed to stdout in the format . + stdin accepts a list of files to watch. + Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully. It will kill the old tasks if a new event is triggered. + USAGE: xnotify [options] [-- [args...]...] VERSION: - 0.2.4 + 0.3.0 GLOBAL OPTIONS: --include value, -i value Include path to watch recursively. - --exclude value, -e value Exclude files from the search using Regular Expression. This only applies to files that were passed as arguments. + --exclude value, -e value Exclude changes from files that match the Regular Expression. This will also apply to events received in server mode. --shallow Disable recursive file globbing. If the path is a directory, the contents will not be included. --listen value Listen on address for file changes e.g. localhost:8080 or just :8080. See --client on how to send file changes. - --base value Use this base path instead of the working directory. This will affect where --include finds the files. If using --listen, it will replace the original base path that was used at the sender. (default: "./") + --base value Use this base path instead of the working directory. This changes the root directory used by --include. If using --listen, it will replace the original base path that was used at the sender. (default: "./") --client value Send file changes to the address e.g. localhost:8080 or just :8080. See --listen on how to receive events. - --batch milliseconds Send the events together if they occur within given milliseconds. The program will only execute given milliseconds after the last event was fired. Only valid with -- argument. (default: 0) - --trigger Run the given command immediately even if there is no file change. Only valid with -- argument. + --batch value Delay emitting all events until it is idle for the given time in milliseconds (also known as debouncing). The --client argument does not support batching. (default: 0) + --terminator value Terminator used to terminate each batch when printing to stdout. Only active when --batch option is used. (default: "\x00") + --trigger Run the given command immediately even if there is no file change. Only valid with the -- argument. --verbose Print verbose logs. --help, -h Print this help. --version, -v print the version @@ -71,25 +78,24 @@ Watch all files in the current directory on the host machine and send events to On the VM: ``` -./xnotify --listen "0.0.0.0:8090" --base "/opt/wwww/project" | xargs -L 1 ./build.sh +./xnotify --listen "0.0.0.0:8090" --base "/home/john/project" | xargs -L 1 ./build.sh ``` You need to set `--base` if the working directory path is different on the host and VM. Remember to use `0.0.0.0` because the traffic is coming from outside the system. -Since the client is triggered using HTTP, you can manually send a request to the client address to trigger a file -change. Send a JSON request in the following format: `{"path": "path/to/file", "operation": "event name"}`. The `operation` +Since the client is triggered using HTTP, you can manually send a request to the client address to trigger an event. +Send a JSON request in the following format: `{"path": "path/to/file", "operation": "event name"}`. The `operation` field is optional as it's only used for logging. Some possible use cases would be triggering a task after a script has finished running, or setting up multiple clients for different events. ### Task Runner Run multiple commands when a file changes. Kills and runs the commands again if a new event comes before the commands -finish. Use `--batch 100` to run the command only 100ms after the last event happened. This will batch multiple -events together and execute the command only once instead of restarting it for every single event. Commands will run in -order as if the `&&` operator is used. Be careful not to run commands that spawn child processes as the child processes -_might not_ terminate with the parent processes. +finish. Commands will run in +the same order as if the `&&` operator is used. Be careful not to run commands that spawn child processes as the child +processes _might not_ terminate with the parent processes. ``` -./xnotify -i . -e "\.git$" --batch 100 -- my_lint arg1 arg2 -- ./compile.sh --flag value -- ./run.sh +./xnotify -i . -e "\.git$" -- my_lint arg1 arg2 -- ./compile.sh --flag value -- ./run.sh ``` This will run the commands in the same manner as: ``` @@ -100,6 +106,33 @@ You can also set the `--trigger` option if you want your command to run immediat ./xnotify -i . --trigger -- run_server.sh 8080 ``` +### Batching + +Sometimes multiple file events are triggered within a very short timespan. This might cause too many processes to +spawn. To solve this we can use the `--batch` argument. This will delay the events from emiting until a certain +duration has passed since the last event — also known as debouncing. For example, by using `--batch 100`, the +events will only be emitted once the last file change is 100ms old. + +Each batch will be terminated with a null character by default. You can change this using `--terminator`. Each event +will still be terminated with new lines. Here are some examples with `xargs`. + +The `-0` or `--null` flags allow `xargs` to recognize each batch using the null character: + +``` +./xnotify -i . --batch 1000 | xargs -0 -L 1 ./build.sh +``` + +Using a different terminator such as `xxx`: +``` +./xnotify -i . --batch 1000 --terminator xxx | xargs -d xxx -L 1 ./build.sh +``` + +Since the events are batched, the `$1` argument will now contain a list of events. You will have to parse this text to +extract the path information if you need it. + +Batching works with the task runner too. It will only restart the tasks after the last event is emitted. The +`--terminator` flag does not apply here. + ## Real World Examples [How to Get File Notification Working on Docker, VirtualBox, VMWare or Vagrant](https://daltontan.com/file-notification-docker-virtualbox-vmware-vagrant/27) diff --git a/go.mod b/go.mod index 3ac2229..4d6ddd3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/AgentCosmic/xnotify require ( github.com/fsnotify/fsnotify v1.4.9 + gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61 gopkg.in/urfave/cli.v1 v1.20.0 ) diff --git a/go.sum b/go.sum index 97bd733..21f1792 100644 --- a/go.sum +++ b/go.sum @@ -2,5 +2,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61 h1:8ajkpB4hXVftY5ko905id+dOnmorcS2CHNxxHLLDcFM= +gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61/go.mod h1:IfMagxm39Ys4ybJrDb7W3Ob8RwxftP0Yy+or/NVz1O8= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= diff --git a/xnotify.go b/xnotify.go index 58c79c0..d74d77c 100644 --- a/xnotify.go +++ b/xnotify.go @@ -15,11 +15,13 @@ import ( "path/filepath" "regexp" "strings" + "sync" "sync/atomic" "time" "github.com/fsnotify/fsnotify" "gopkg.in/urfave/cli.v1" + "gopkg.in/alessio/shellescape.v1" ) // Event for each file change @@ -31,19 +33,27 @@ type Event struct { type program struct { eventChannel chan Event // track file change events - tasks [][]string - process *os.Process - processChannel chan bool // track the process we are spawning clientAddress string batchMS int - batchSize int32 base string defaultBase string - trigger bool - hasTasks bool excludePatterns []string + // used for print runner + terminator string // used to terminate each batch + mu sync.Mutex + timer *time.Timer // used for debouncing + batchEvents []Event // collect events for next batch + // used for task runner + trigger bool // whether to trigger tasks immediately on startup + hasTasks bool // if there is any task to run + batchSize int32 // keep track of the last event to trigger the runner + tasks [][]string // tasks to run + process *os.Process // task process + processChannel chan bool // track the process we are spawning } +const NullChar = "\000" + func main() { log.SetPrefix("[xnotify] ") log.SetFlags(0) @@ -54,11 +64,11 @@ func main() { } app := cli.NewApp() app.Name = "xnotify" - app.Version = "0.2.4" + app.Version = "0.3.0" app.Usage = "Watch files for changes." + "\n File changes will be printed to stdout in the format ." + "\n stdin accepts a list of files to watch." + - "\n Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully." + "\n Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully. It will kill the old tasks if a new event is triggered." app.UsageText = "xnotify [options] [-- [args...]...]" app.HideHelp = true app.Flags = []cli.Flag{ @@ -68,7 +78,7 @@ func main() { }, cli.StringSliceFlag{ Name: "exclude, e", - Usage: "Exclude changes from files that match the Regular Expression. This will also apply to events recieved in server mode.", + Usage: "Exclude changes from files that match the Regular Expression. This will also apply to events received in server mode.", }, cli.BoolFlag{ Name: "shallow", @@ -81,7 +91,7 @@ func main() { cli.StringFlag{ Name: "base", Value: prog.defaultBase, - Usage: "Use this base path instead of the working directory. This will affect where --include finds the files. If using --listen, it will replace the original base path that was used at the sender.", + Usage: "Use this base path instead of the working directory. This changes the root directory used by --include. If using --listen, it will replace the original base path that was used at the sender.", Destination: &prog.base, }, cli.StringFlag{ @@ -91,12 +101,18 @@ func main() { }, cli.IntFlag{ Name: "batch", - Usage: "Send the events together if they occur within given `milliseconds`. The program will only execute given milliseconds after the last event was fired. Only valid with -- argument.", + Usage: "Delay emitting all events until it is idle for the given time in milliseconds (also known as debouncing). The --client argument does not support batching.", Destination: &prog.batchMS, }, + cli.StringFlag{ + Name: "terminator", + Usage: "Terminator used to terminate each batch when printing to stdout. Only active when --batch option is used.", + Destination: &prog.terminator, + Value: NullChar, + }, cli.BoolFlag{ Name: "trigger", - Usage: "Run the given command immediately even if there is no file change. Only valid with -- argument.", + Usage: "Run the given command immediately even if there is no file change. Only valid with the -- argument.", Destination: &prog.trigger, }, cli.BoolFlag{ @@ -148,8 +164,8 @@ func (prog *program) action(c *cli.Context) (err error) { if prog.base != prog.defaultBase && c.String("listen") == "" { noEffect("base") } - if prog.batchMS != 0 && !prog.hasTasks { - noEffect("batch") + if prog.terminator != NullChar && prog.batchMS == 0 { + noEffect("terminator") } if prog.trigger && !prog.hasTasks { noEffect("trigger") @@ -367,11 +383,6 @@ func (prog *program) fileChanged(e Event) { } } -// runner that prints to stdout -func (prog *program) printRunner(e Event) { - fmt.Printf("%s %s\n", e.Operation, e.Path) -} - // runner that sends to a another client via http func (prog *program) httpRunner(e Event) { b, err := json.Marshal(&e) @@ -392,6 +403,29 @@ func (prog *program) httpRunner(e Event) { } } +// runner that prints to stdout +func (prog *program) printRunner(e Event) { + prog.mu.Lock() + defer prog.mu.Unlock() + dur, err := time.ParseDuration(fmt.Sprint(prog.batchMS, "ms")) + if err != nil { + panic(err) + } + if prog.timer != nil { + prog.timer.Stop() + } + prog.batchEvents = append(prog.batchEvents, e) + prog.timer = time.AfterFunc(dur, func() { + for _, e := range prog.batchEvents { + fmt.Printf("%s %s\n", e.Operation, shellescape.Quote(e.Path)) + } + if dur > 0 { + fmt.Print(prog.terminator) + } + prog.batchEvents = make([]Event, 0) + }) +} + // // ----- Batch program runner ----- // @@ -405,6 +439,7 @@ func (prog *program) programRunner(eventChannel chan Event, e Event) { } atomic.AddInt32(&prog.batchSize, 1) time.AfterFunc(dur, func() { + // only need to execute once the last task is here, means the batchMS time has passed since last event if atomic.LoadInt32(&prog.batchSize) == 1 { // if program is already done, there will be no effect if prog.process != nil { @@ -415,6 +450,7 @@ func (prog *program) programRunner(eventChannel chan Event, e Event) { // need to clear anything from previous run <-prog.processChannel } + // tell the loop there's new event eventChannel <- e // wait until the process is captured before proceeding so we can kill it later <-prog.processChannel