diff --git a/.gitignore b/.gitignore index b67bd86..a5bf82a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ dist/ # compiled files zap +# Include our actual code, which is excluded by the "zap" directive above. +!/cmd/zap + # Goland .idea/* diff --git a/.travis.yml b/.travis.yml index 10a32dd..fde875b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,8 +39,8 @@ os: script: - diff -u <(echo -n) <(go fmt $(go list ./...)) - go vet $(go list ./...) - - go test -short -v ./... -race -coverprofile=coverage.txt -covermode=atomic - - go build -v ./... + - go test -short -v ./... -race -coverprofile=coverage.txt -covermode=atomic ./cmd + - go build -o zap -v ./cmd/ - ./e2e.sh # calls goreleaser diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1dec68..d23a833 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ ## Contributing to Zap Patches are welcome! Please use the standard GitHub workflow - fork this -repo and submit a PR. I'll usually get to it within a few days. +repo and submit a PR. I'll usually get to it within a few days. If I miss your PR email, feel free to ping me directly at isgsmirnov@gmail.com after about a week. ## Setting up dev environment @@ -13,16 +13,16 @@ or your favorite web editor with Golang support. git clone $your_fork zap cd zap go get -go build . # sanity check +go build -o zap -v ./cmd/ # sanity check # install test deps and run all tests -go test ./... -v +go test -short -v ./... -race -coverprofile=coverage.txt -covermode=atomic ./cmd ./e2e.sh ``` ## Handy commands for local development: -- `go build && ./zap` to run locally +- `go build -o zap -v ./cmd/ && ./zap` to run locally - `curl -I -L -H 'Host: g' localhost:8927/z` - to test locally e2e - `goconvey -excludedDirs dist` - launches web UI for go tests. - `./e2e.sh` runs CLI tests. diff --git a/main.go b/cmd/main.go similarity index 68% rename from main.go rename to cmd/main.go index d7815b7..749adee 100644 --- a/main.go +++ b/cmd/main.go @@ -8,6 +8,8 @@ import ( "os" "path" + "github.com/issmirnov/zap/cmd/zap" + "github.com/fsnotify/fsnotify" "github.com/julienschmidt/httprouter" @@ -34,7 +36,7 @@ func main() { } // load config for first time. - c, err := parseYaml(*configName) + c, err := zap.ParseYaml(*configName) if err != nil { log.Printf("Error parsing config file. Please fix syntax: %s\n", err) return @@ -42,7 +44,7 @@ func main() { // Perform extended validation of config. if *validate { - if err := validateConfig(c); err != nil { + if err := zap.ValidateConfig(c); err != nil { fmt.Println(err.Error()) os.Exit(1) } @@ -50,8 +52,8 @@ func main() { os.Exit(0) } - context := &context{config: c} - updateHosts(context) // sync changes since last run. + context := &zap.Context{Config: c} + zap.UpdateHosts(context) // sync changes since last run. // Enable hot reload. watcher, err := fsnotify.NewWatcher() @@ -60,26 +62,30 @@ func main() { } defer watcher.Close() - // Enable hot reload. - cb := makeCallback(context, *configName) - go watchChanges(watcher, *configName, cb) + cb := zap.MakeReloadCallback(context, *configName) + go zap.WatchConfigFileChanges(watcher, *configName, cb) err = watcher.Add(path.Dir(*configName)) if err != nil { log.Fatal(err) } // Set up routes. + router := SetupRouter(context) + + // TODO check for errors - addr in use, sudo issues, etc. + fmt.Printf("Launching %s on %s:%d\n", appName, *host, *port) + log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", *host, *port), router)) +} + +func SetupRouter(context *zap.Context) *httprouter.Router { router := httprouter.New() - router.Handler("GET", "/", ctxWrapper{context, IndexHandler}) - router.Handler("GET", "/varz", ctxWrapper{context, VarsHandler}) - router.HandlerFunc("GET", "/healthz", HealthHandler) + router.Handler("GET", "/", zap.CtxWrapper{Context: context, H: zap.IndexHandler}) + router.Handler("GET", "/varz", zap.CtxWrapper{Context: context, H: zap.VarsHandler}) + router.HandlerFunc("GET", "/healthz", zap.HealthHandler) // https://github.com/julienschmidt/httprouter is having issues with // wildcard handling. As a result, we have to register index handler // as the fallback. Fix incoming. - router.NotFound = ctxWrapper{context, IndexHandler} - - // TODO check for errors - addr in use, sudo issues, etc. - fmt.Printf("Launching %s on %s:%d\n", appName, *host, *port) - log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", *host, *port), router)) + router.NotFound = zap.CtxWrapper{Context: context, H: zap.IndexHandler} + return router } diff --git a/config.go b/cmd/zap/config.go similarity index 79% rename from config.go rename to cmd/zap/config.go index 39d4bf8..e092fe2 100644 --- a/config.go +++ b/cmd/zap/config.go @@ -1,4 +1,4 @@ -package main +package zap import ( "bytes" @@ -47,8 +47,8 @@ func parseYamlString(config string) (*gabs.Container, error) { return j, nil } -// parseYaml takes a file name and returns a gabs config object. -func parseYaml(fname string) (*gabs.Container, error) { +// ParseYaml takes a file name and returns a gabs Config object. +func ParseYaml(fname string) (*gabs.Container, error) { data, err := Afero.ReadFile(fname) if err != nil { fmt.Printf("Unable to read file: %s\n", err.Error()) @@ -63,10 +63,10 @@ func parseYaml(fname string) (*gabs.Container, error) { return j, nil } -// validateConfig verifies that there are no unexpected values in the config file. -// at each level of the config, we should either have a KV for expansions, or a leaf node +// ValidateConfig verifies that there are no unexpected values in the Config file. +// at each level of the Config, we should either have a KV for expansions, or a leaf node // with the values oneof "expand", "query", "ssl_off" that map to a string. -func validateConfig(c *gabs.Container) error { +func ValidateConfig(c *gabs.Container) error { var errors *multierror.Error children := c.ChildrenMap() seenKeys := make(map[string]struct{}) @@ -105,7 +105,7 @@ func validateConfig(c *gabs.Container) error { errors = multierror.Append(errors, fmt.Errorf("unexpected string value under key %s, got: %v", k, v.Data())) } // recurse, collect any errors. - if err := validateConfig(v); err != nil { + if err := ValidateConfig(v); err != nil { errors = multierror.Append(errors, err) } } @@ -113,7 +113,9 @@ func validateConfig(c *gabs.Container) error { return errors.ErrorOrNil() } -func watchChanges(watcher *fsnotify.Watcher, fname string, cb func()) { +// WatchConfigFileChanges will attach an fsnotify watcher to the config file, and trigger +// the cb function when the file is updated. +func WatchConfigFileChanges(watcher *fsnotify.Watcher, fname string, cb func()) { for { select { case event := <-watcher.Events: @@ -133,22 +135,22 @@ func watchChanges(watcher *fsnotify.Watcher, fname string, cb func()) { } // TODO: add tests. simulate touching a file. -// updateHosts will attempt to write the zap list of shortcuts +// UpdateHosts will attempt to write the zap list of shortcuts // to /etc/hosts. It will gracefully fail if there are not enough // permissions to do so. -func updateHosts(c *context) { +func UpdateHosts(c *Context) { hostPath := "/etc/hosts" // 1. read file, prep buffer. data, err := ioutil.ReadFile(hostPath) if err != nil { - log.Println("open config: ", err) + log.Println("open Config: ", err) } var replacement bytes.Buffer // 2. generate payload. replacement.WriteString(delimStart) - children := c.config.ChildrenMap() + children := c.Config.ChildrenMap() for k := range children { replacement.WriteString(fmt.Sprintf("127.0.0.1 %s\n", k)) } @@ -170,27 +172,27 @@ func updateHosts(c *context) { } } -// makeCallback returns a func that that updates global state. -func makeCallback(c *context, configName string) func() { +// MakeReloadCallback returns a func that that reads the config file and updates global state. +func MakeReloadCallback(c *Context, configName string) func() { return func() { - data, err := parseYaml(configName) + data, err := ParseYaml(configName) if err != nil { - log.Printf("Error loading new config: %s. Fallback to old config.", err) + log.Printf("Error loading new Config: %s. Fallback to old Config.", err) return } - err = validateConfig(data) + err = ValidateConfig(data) if err != nil { - log.Printf("Error validating new config: %s. Fallback to old config.", err) + log.Printf("Error validating new Config: %s. Fallback to old Config.", err) return } - // Update config atomically - c.configMtx.Lock() - c.config = data - c.configMtx.Unlock() + // Update Config atomically + c.ConfigMtx.Lock() + c.Config = data + c.ConfigMtx.Unlock() // Sync DNS entries. - updateHosts(c) + UpdateHosts(c) return } } diff --git a/config_test.go b/cmd/zap/config_test.go similarity index 85% rename from config_test.go rename to cmd/zap/config_test.go index a7908f9..a4a77ed 100644 --- a/config_test.go +++ b/cmd/zap/config_test.go @@ -1,4 +1,4 @@ -package main +package zap import ( "testing" @@ -120,8 +120,8 @@ func TestParseYaml(t *testing.T) { Convey("Given a valid 'c.yml' file", t, func() { Afero = &afero.Afero{Fs: afero.NewMemMapFs()} Afero.WriteFile("c.yml", []byte(cYaml), 0644) - c, err := parseYaml("c.yml") - Convey("parseYaml should throw no error", func() { + c, err := ParseYaml("c.yml") + Convey("ParseYaml should throw no error", func() { So(err, ShouldBeNil) }) Convey("the gabs object should have path 'zz' present", func() { @@ -131,33 +131,33 @@ func TestParseYaml(t *testing.T) { } func TestValidateConfig(t *testing.T) { - Convey("Given a correctly formatted yaml config", t, func() { + Convey("Given a correctly formatted yaml Config", t, func() { conf, _ := parseYamlString(cYaml) //fmt.Printf(err.Error()) Convey("The validator should pass", func() { - So(validateConfig(conf), ShouldBeNil) + So(ValidateConfig(conf), ShouldBeNil) }) }) // The YAML libraries don't have support for detecting duplicate keys // at parse time. Users will have to figure this out themselves. - //Convey("Given a yaml config with duplicated keys", t, func() { + //Convey("Given a yaml Config with duplicated keys", t, func() { // conf, _ := parseYamlString(duplicatedYAML) // Convey("The validator should complain", func() { - // So(validateConfig(conf), ShouldNotBeNil) + // So(ValidateConfig(conf), ShouldNotBeNil) // }) //}) - Convey("Given a YAML config with unknown keys", t, func() { + Convey("Given a YAML Config with unknown keys", t, func() { conf, _ := parseYamlString(badkeysYAML) Convey("The validator should raise an error", func() { - So(validateConfig(conf), ShouldNotBeNil) + So(ValidateConfig(conf), ShouldNotBeNil) }) }) - Convey("Given a YAML config with malformed values", t, func() { + Convey("Given a YAML Config with malformed values", t, func() { conf, _ := parseYamlString(badValuesYAML) - err := validateConfig(conf) + err := ValidateConfig(conf) Convey("The validator should raise a ton of errors", func() { So(err, ShouldNotBeNil) So(err.Error(), ShouldContainSubstring, "expected float64 value for string, got: not_int") diff --git a/cmd/zap/structs.go b/cmd/zap/structs.go new file mode 100644 index 0000000..2659a8f --- /dev/null +++ b/cmd/zap/structs.go @@ -0,0 +1,35 @@ +package zap + +import ( + "fmt" + "net/http" + "sync" + + "github.com/Jeffail/gabs/v2" +) + +type Context struct { + // Json container with path configs + Config *gabs.Container + + // Enables safe hot reloading of Config. + ConfigMtx sync.Mutex +} + +type CtxWrapper struct { + *Context + H func(*Context, http.ResponseWriter, *http.Request) (int, error) +} + +func (cw CtxWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { + status, err := cw.H(cw.Context, w, r) // this runs the actual handler, defined in struct. + if err != nil { + switch status { + case http.StatusInternalServerError: + http.Error(w, fmt.Sprintf("HTTP %d: %q", status, err), status) + // TODO - add bad request? + default: + http.Error(w, err.Error(), status) + } + } +} diff --git a/text.go b/cmd/zap/text.go similarity index 94% rename from text.go rename to cmd/zap/text.go index d5a16a4..9112abf 100644 --- a/text.go +++ b/cmd/zap/text.go @@ -1,4 +1,4 @@ -package main +package zap import ( "bytes" @@ -59,11 +59,11 @@ func getPrefix(c *gabs.Container) (string, int, error) { return "", 0, fmt.Errorf("casting port key to float64 failed for %T:%v", p, p) } - return "", 0, fmt.Errorf("error in config, no key matching 'expand', 'query', 'port' or 'schema' in %s", c.String()) + return "", 0, fmt.Errorf("error in Config, no key matching 'expand', 'query', 'port' or 'schema' in %s", c.String()) } -// expandPath takes a config, list of tokens (parsed from request) and the results buffer -// At each level of recursion, it matches the token to the action described in the config, and writes it +// expandPath takes a Config, list of tokens (parsed from request) and the results buffer +// At each level of recursion, it matches the token to the action described in the Config, and writes it // to the result buffer. There is special care needed to handle slashes correctly, which makes this function // quite nontrivial. Tests are crucial to ensure correctness. func expandPath(c *gabs.Container, token *list.Element, res *bytes.Buffer) { diff --git a/text_test.go b/cmd/zap/text_test.go similarity index 99% rename from text_test.go rename to cmd/zap/text_test.go index 16dd7d8..344d5dc 100644 --- a/text_test.go +++ b/cmd/zap/text_test.go @@ -1,4 +1,4 @@ -package main +package zap import ( "bytes" diff --git a/web.go b/cmd/zap/web.go similarity index 58% rename from web.go rename to cmd/zap/web.go index 7565dae..d365740 100644 --- a/web.go +++ b/cmd/zap/web.go @@ -1,45 +1,18 @@ -package main +package zap import ( "bytes" "fmt" "io" "net/http" - "sync" "encoding/json" "github.com/Jeffail/gabs/v2" ) -type context struct { - // Json container with path configs - config *gabs.Container - - // Enables safe hot reloading of config. - configMtx sync.Mutex -} - -type ctxWrapper struct { - *context - H func(*context, http.ResponseWriter, *http.Request) (int, error) -} - -func (cw ctxWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { - status, err := cw.H(cw.context, w, r) // this runs the actual handler, defined in struct. - if err != nil { - switch status { - case http.StatusInternalServerError: - http.Error(w, fmt.Sprintf("HTTP %d: %q", status, err), status) - // TODO - add bad request? - default: - http.Error(w, err.Error(), status) - } - } -} - // IndexHandler handles all the non status expansions. -func IndexHandler(ctx *context, w http.ResponseWriter, r *http.Request) (int, error) { +func IndexHandler(ctx *Context, w http.ResponseWriter, r *http.Request) (int, error) { var host string if r.Header.Get("X-Forwarded-Host") != "" { host = r.Header.Get("X-Forwarded-Host") @@ -50,17 +23,17 @@ func IndexHandler(ctx *context, w http.ResponseWriter, r *http.Request) (int, er var hostConfig *gabs.Container var ok bool - // Check if host present in config. - children := ctx.config.ChildrenMap() + // Check if host present in Config. + children := ctx.Config.ChildrenMap() if hostConfig, ok = children[host]; !ok { - return 404, fmt.Errorf("Shortcut '%s' not found in config.", host) + return 404, fmt.Errorf("Shortcut '%s' not found in Config.", host) } tokens := tokenize(host + r.URL.Path) - // Set up handles on token and config. We might need to skip ahead if there's a custom schema set. + // Set up handles on token and Config. We might need to skip ahead if there's a custom schema set. tokensStart := tokens.Front() - conf := ctx.config + conf := ctx.Config var path bytes.Buffer if s := hostConfig.Path(sslKey).Data(); s != nil && s.(bool) { @@ -89,10 +62,10 @@ func HealthHandler(w http.ResponseWriter, r *http.Request) { io.WriteString(w, `OK`) } -// VarsHandler responds to /varz request and prints config. -func VarsHandler(c *context, w http.ResponseWriter, r *http.Request) (int, error) { +// VarsHandler responds to /varz request and prints Config. +func VarsHandler(c *Context, w http.ResponseWriter, r *http.Request) (int, error) { w.WriteHeader(http.StatusOK) - io.WriteString(w, jsonPrettyPrint(c.config.String())) + io.WriteString(w, jsonPrettyPrint(c.Config.String())) return 200, nil } diff --git a/web_test.go b/cmd/zap/web_test.go similarity index 94% rename from web_test.go rename to cmd/zap/web_test.go index 7887a0f..2a8342a 100644 --- a/web_test.go +++ b/cmd/zap/web_test.go @@ -1,4 +1,4 @@ -package main +package zap import ( "net/http" @@ -11,16 +11,16 @@ import ( . "github.com/smartystreets/goconvey/convey" ) -// TODO: add tests that use erroneous config. +// TODO: add tests that use erroneous Config. // Will likely require injecting custom logger and intercepting error msgs. // See https://elithrar.github.io/article/testing-http-handlers-go/ for comments. func TestIndexHandler(t *testing.T) { - Convey("Given app is set up with default config", t, func() { + Convey("Given app is set up with default Config", t, func() { c, err := loadTestYaml() So(err, ShouldBeNil) - context := &context{config: c} - appHandler := &ctxWrapper{context, IndexHandler} + context := &Context{Config: c} + appHandler := &CtxWrapper{context, IndexHandler} handler := http.Handler(appHandler) Convey("When we GET http://g/z", func() { req, err := http.NewRequest("GET", "/z", nil) @@ -247,7 +247,7 @@ func TestIndexHandler(t *testing.T) { }) }) - Convey("When we GET http://ch/foobar with schema set to 'chrome' where 'foobar' isn't in the config ", func() { + Convey("When we GET http://ch/foobar with schema set to 'chrome' where 'foobar' isn't in the Config ", func() { req, err := http.NewRequest("GET", "/foobar", nil) So(err, ShouldBeNil) req.Host = "ch" @@ -295,12 +295,12 @@ func TestIndexHandler(t *testing.T) { }) } -// BenchmarkIndexHandler tests request processing geed when context is preloaded. +// BenchmarkIndexHandler tests request processing speed when Context is preloaded. // Run with go test -run=BenchmarkIndexHandler -bench=. // results: 500000x 2555 ns/op func BenchmarkIndexHandler(b *testing.B) { c, _ := loadTestYaml() - context := &context{config: c} - appHandler := &ctxWrapper{context, IndexHandler} + context := &Context{Config: c} + appHandler := &CtxWrapper{context, IndexHandler} handler := http.Handler(appHandler) req, _ := http.NewRequest("GET", "/z", nil) req.Host = "g" @@ -329,12 +329,12 @@ func TestHealthCheckHandler(t *testing.T) { } func TestVarzHandler(t *testing.T) { - Convey("Given app is set up with default config", t, func() { + Convey("Given app is set up with default Config", t, func() { c, err := loadTestYaml() So(err, ShouldBeNil) - context := &context{config: c} + context := &Context{Config: c} - appHandler := &ctxWrapper{context, VarsHandler} + appHandler := &CtxWrapper{context, VarsHandler} handler := http.Handler(appHandler) Convey("When we GET /varz", func() { req, err := http.NewRequest("GET", "/varz", nil) @@ -351,17 +351,17 @@ func TestVarzHandler(t *testing.T) { _, err := yaml.YAMLToJSON(rr.Body.Bytes()) So(err, ShouldBeNil) }) - Convey("It should equal the config file", func() { + Convey("It should equal the Config file", func() { conf, err := yaml.YAMLToJSON(c.Bytes()) - So(err, ShouldBeNil) // sanity check. + So(err, ShouldBeNil) resp, err := yaml.YAMLToJSON(rr.Body.Bytes()) - So(err, ShouldBeNil) // sanity check. + So(err, ShouldBeNil) + // This does not work: "So(resp, ShouldEqual, []byte(jsonPrettyPrint(string(conf))))" // We get a nicely formatted response, but when we feed it into YAMLToJSON it collapses our nice // newlines. As a result, directly comparing the byte arrays here is a nogo. Therefore, we cheat // and utilize the separately tested jsonPrettyPrint to idempotently indent the JSON and compare that. - // This does not work: "So(resp, ShouldEqual, []byte(jsonPrettyPrint(string(conf))))" So(jsonPrettyPrint(string(resp)), ShouldEqual, jsonPrettyPrint(string(conf))) }) })