From 25ae6457831c1777f474830c50e90b77fa70b7bf Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Sat, 7 Dec 2019 12:40:53 +0200 Subject: [PATCH 1/6] Reduce some allocations. --- config.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- config_test.go | 35 ++++++++++++++++++++--------------- loaders.go | 33 --------------------------------- testdata/.env | 6 +++--- vars.go | 7 +++---- 5 files changed, 72 insertions(+), 59 deletions(-) delete mode 100644 loaders.go diff --git a/config.go b/config.go index 5271314..764599d 100644 --- a/config.go +++ b/config.go @@ -10,8 +10,10 @@ package config import ( + "bytes" "encoding/json" "errors" + "fmt" "os" ) @@ -23,6 +25,11 @@ type Loader struct { i interface{} } +// Load creates a Loader with given struct +func Load(i interface{}) *Loader { + return &Loader{i: i} +} + // Env loads config into struct from environment variables func (l *Loader) Env() error { if err := checkNilStruct(l.i); err != nil { @@ -43,7 +50,33 @@ func (l *Loader) EnvFile(files ...string) error { files = append(files, dotEnvFile) } - return fromEnvFile(l.i, files...) + vars := make(map[string]string) + + for i := 0; i < len(files); i++ { + f, err := os.Open(files[i]) + if err != nil { + return fmt.Errorf("config: %s", err) + } + + err = parseVars(f, vars) + + if err != nil { + if e := f.Close(); e != nil { + return fmt.Errorf("config: %s", e) + } + return fmt.Errorf("config: %s", err) + } + + if err = f.Close(); err != nil { + return fmt.Errorf("config: %s", err) + } + } + + f := func(s string) string { + return vars[s] + } + + return parseIntoStruct(l.i, f) } // Bytes loads config into struct from byte array @@ -81,7 +114,16 @@ func checkNilStruct(i interface{}) error { return nil } -// Load creates a Loader with given struct -func Load(i interface{}) *Loader { - return &Loader{i: i} +func fromBytes(i interface{}, input []byte) error { + vars := make(map[string]string) + err := parseVars(bytes.NewReader(input), vars) + if err != nil { + return err + } + + f := func(s string) string { + return vars[s] + } + + return parseIntoStruct(i, f) } diff --git a/config_test.go b/config_test.go index b7b984a..5224203 100644 --- a/config_test.go +++ b/config_test.go @@ -58,11 +58,13 @@ func TestEnv(t *testing.T) { t.Fatal(err) } - kv, err := vars(bytes.NewReader(input)) + vars := make(map[string]string) + err = parseVars(bytes.NewReader(input), vars) if err != nil { t.Fatal(err) } - for k, v := range kv { + + for k, v := range vars { if err := os.Setenv(k, v); err != nil { t.Fatalf(`cannot set env variable "%s" with value "%s": "%s"`, k, v, err) } @@ -77,7 +79,7 @@ func TestEnv(t *testing.T) { t.Errorf("\nhave: %v\nwant: %v", actual, expected) } - for k := range kv { + for k := range vars { if err := os.Unsetenv(k); err != nil { t.Fatal(err) } @@ -144,7 +146,7 @@ func TestString(t *testing.T) { } func TestJson(t *testing.T) { - input := json.RawMessage(`{ + input := json.RawMessage(`{ "StructPtr":null, "String":" string\\\" ", "A":1, @@ -161,24 +163,24 @@ func TestJson(t *testing.T) { "F32":15425.2231, "F64":245232212.9844448, "IsSet":true, - "Redis":{ - "Connection":{ + "Redis":{ + "Connection":{ "Host":" localhost ", "Port":6379 } }, "Timeout":2000000000, - "Mongo":{ - "Database":{ + "Mongo":{ + "Database":{ "Host":"mongodb://user:pass==@host.tld:955/?ssl=true&replicaSet=globaldb", - "Collection":{ + "Collection":{ "Name":"dXM9ZXJz", "Other":1, "X":97 } } }, - "Struct":{ + "Struct":{ "Field":"Value" } }`) @@ -269,25 +271,28 @@ func (e *errReader) Read(p []byte) (n int, err error) { } func TestWithParseReaderError(t *testing.T) { - v, err := vars(&errReader{}) - if v != nil { - t.Error("expected nil map") + kv := make(map[string]string) + err := parseVars(&errReader{}, kv) + if len(kv) > 0 { + t.Error("expected empty map") } if err == nil { t.Error("expected reader error") } } -// BenchmarkVars-8 1478143 856 ns/op 4144 B/op 2 allocs/op +// BenchmarkVars-8 1663723 749 ns/op 4096 B/op 1 allocs/op func BenchmarkVars(b *testing.B) { + b.ReportAllocs() input, _, err := testdata() if err != nil { b.Fatal(err) } + vars := make(map[string]string) reader := bytes.NewReader(input) for n := 0; n < b.N; n++ { - _, err := vars(reader) + err := parseVars(reader, vars) if err != nil { b.Fatal(err) } diff --git a/loaders.go b/loaders.go deleted file mode 100644 index 1f951db..0000000 --- a/loaders.go +++ /dev/null @@ -1,33 +0,0 @@ -package config - -import ( - "bytes" - "fmt" - "io/ioutil" -) - -func fromEnvFile(i interface{}, files ...string) error { - var input []byte - for i := 0; i < len(files); i++ { - data, err := ioutil.ReadFile(files[i]) - if err != nil { - return fmt.Errorf("config: %s", err) - } - input = append(input, data...) - } - - return fromBytes(i, input) -} - -func fromBytes(i interface{}, input []byte) error { - v, err := vars(bytes.NewReader(input)) - if err != nil { - return err - } - - f := func(s string) string { - return v[s] - } - - return parseIntoStruct(i, f) -} diff --git a/testdata/.env b/testdata/.env index 2e14a70..c4f4030 100644 --- a/testdata/.env +++ b/testdata/.env @@ -1,8 +1,8 @@ # key=value TIMEOUT=2000000000 -ABC=" string\" " -A=1 -B=2 +ABC =" string\" " +A =1 + B =2 C=3# key=value D =4 E=5 diff --git a/vars.go b/vars.go index 84095e5..720c158 100644 --- a/vars.go +++ b/vars.go @@ -8,9 +8,8 @@ import ( "unicode" ) -func vars(r io.Reader) (map[string]string, error) { +func parseVars(r io.Reader, vars map[string]string) error { reader := bufio.NewReader(r) - vars := make(map[string]string) var name, value []byte @@ -28,7 +27,7 @@ func vars(r io.Reader) (map[string]string, error) { break } - return nil, fmt.Errorf("config: cannot read from input (%s)", err) + return fmt.Errorf("config: cannot read from input (%s)", err) } if r == '#' { @@ -88,7 +87,7 @@ func vars(r io.Reader) (map[string]string, error) { } } - return vars, nil + return nil } func varName(n []byte) string { From 3dda10c5fd8c971d99e9230112689f59f3bef024 Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Sat, 7 Dec 2019 12:50:57 +0200 Subject: [PATCH 2/6] Remove unnecessary calls. --- config.go | 4 ++-- vars.go | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/config.go b/config.go index 764599d..0f52e4b 100644 --- a/config.go +++ b/config.go @@ -116,8 +116,8 @@ func checkNilStruct(i interface{}) error { func fromBytes(i interface{}, input []byte) error { vars := make(map[string]string) - err := parseVars(bytes.NewReader(input), vars) - if err != nil { + + if err := parseVars(bytes.NewReader(input), vars); err != nil { return err } diff --git a/vars.go b/vars.go index 720c158..6432a4e 100644 --- a/vars.go +++ b/vars.go @@ -22,7 +22,7 @@ func parseVars(r io.Reader, vars map[string]string) error { if err != nil { if err == io.EOF { if atValue { - vars[varName(name)] = varValue(value) + vars[string(name)] = varValue(value) } break } @@ -32,7 +32,7 @@ func parseVars(r io.Reader, vars map[string]string) error { if r == '#' { if atValue { - vars[varName(name)] = varValue(value) + vars[string(name)] = varValue(value) } name = nil @@ -45,7 +45,7 @@ func parseVars(r io.Reader, vars map[string]string) error { if r == '\n' || r == '\r' { if atValue { - vars[varName(name)] = varValue(value) + vars[string(name)] = varValue(value) } name = nil @@ -90,10 +90,6 @@ func parseVars(r io.Reader, vars map[string]string) error { return nil } -func varName(n []byte) string { - return string(bytes.TrimSpace(n)) -} - func varValue(v []byte) string { return string(bytes.Trim(bytes.TrimSpace(v), `"'`)) } From 38c72b903e8504da63c25d6e77fc6f050ba02c5e Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Sat, 7 Dec 2019 13:08:32 +0200 Subject: [PATCH 3/6] Simplify lint. --- .gitignore | 1 - Makefile | 17 +++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index b1b4513..b69c78c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ .idea -bin cover.out diff --git a/Makefile b/Makefile index 49428c1..f42ac82 100644 --- a/Makefile +++ b/Makefile @@ -5,26 +5,23 @@ GO111MODULE=on all: test lint -GOLINT := $(shell which golint) - test: go test -cover -v ./... bench: go test -bench=. -benchmem -v -run=Bench ./... -lint: -ifndef GOLINT - GO111MODULE=off && go get -u golang.org/x/lint/golint -endif +lint: check-lint golint -set_exit_status ./... - - @[ ! -f ./bin/golangci-lint ] && curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh \ - | sh -s -- -b ./bin v1.21.0 || true - ./bin/golangci-lint run + golangci-lint run coverage: go test -v -coverprofile $(COVER_PROFILE) ./... && go tool cover -html=$(COVER_PROFILE) prepushhook: echo '#!/bin/sh\n\nmake' > .git/hooks/pre-push && chmod +x .git/hooks/pre-push + +check-lint: + @[ $(shell which golint) ] || (GO111MODULE=off && go get -u golang.org/x/lint/golint) + @[ $(shell which golangci-lint) ] || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh \ + | sh -s -- -b $(shell go env GOPATH)/bin v1.21.0 From 5817fc21584c68152a587d193bb3dbbe5838fb44 Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Sun, 8 Dec 2019 12:32:40 +0200 Subject: [PATCH 4/6] Interpolate vars. --- .golangci.yml | 2 +- config.go | 4 +++ config_test.go | 47 ++++++++++++++++++--------------- testdata/.env | 5 +++- vars.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 23 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 7985e2d..0d34a53 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -42,7 +42,7 @@ linters-settings: min-confidence: 0.8 gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 17 + min-complexity: 25 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/config.go b/config.go index 0f52e4b..0a94db2 100644 --- a/config.go +++ b/config.go @@ -72,6 +72,8 @@ func (l *Loader) EnvFile(files ...string) error { } } + interpolateVars(vars) + f := func(s string) string { return vars[s] } @@ -121,6 +123,8 @@ func fromBytes(i interface{}, input []byte) error { return err } + interpolateVars(vars) + f := func(s string) string { return vars[s] } diff --git a/config_test.go b/config_test.go index 5224203..48a03fa 100644 --- a/config_test.go +++ b/config_test.go @@ -28,24 +28,25 @@ type Config struct { Port int `env:"REDIS_PORT"` } } - String string `env:"ABC"` - Struct Struct - StructPtr *Struct - D int64 - E int - ENeg int `env:"E_NEG"` - UD uint64 - UE uint - F64 float64 - Timeout time.Duration - C int32 - UC uint32 - F32 float32 - B int16 - UB uint16 - A int8 - UA uint8 - IsSet bool + String string `env:"ABC"` + Struct Struct + StructPtr *Struct + D int64 + E int + ENeg int `env:"E_NEG"` + UD uint64 + UE uint + F64 float64 + Timeout time.Duration + C int32 + UC uint32 + F32 float32 + B int16 + UB uint16 + A int8 + UA uint8 + IsSet bool + Interpolated string } type Struct struct { @@ -63,6 +64,7 @@ func TestEnv(t *testing.T) { if err != nil { t.Fatal(err) } + interpolateVars(vars) for k, v := range vars { if err := os.Setenv(k, v); err != nil { @@ -182,7 +184,8 @@ func TestJson(t *testing.T) { }, "Struct":{ "Field":"Value" - } + }, + "Interpolated":"$B env_1 $ $B \\3" }`) _, expected, err := testdata() @@ -281,9 +284,10 @@ func TestWithParseReaderError(t *testing.T) { } } -// BenchmarkVars-8 1663723 749 ns/op 4096 B/op 1 allocs/op -func BenchmarkVars(b *testing.B) { +// Benchmark_parseVars-8 1663723 749 ns/op 4096 B/op 1 allocs/op +func Benchmark_parseVars(b *testing.B) { b.ReportAllocs() + input, _, err := testdata() if err != nil { b.Fatal(err) @@ -367,6 +371,7 @@ func testdata() ([]byte, Config, error) { X: 'a', }, }}, + Interpolated: "$B env_1 $ $B \\3", } return input, expected, nil diff --git a/testdata/.env b/testdata/.env index c4f4030..d9e927a 100644 --- a/testdata/.env +++ b/testdata/.env @@ -11,6 +11,7 @@ UA=1 # key=value UB=2 # comment UC=3 + UD=4 UE=5 F32=15425.2231 @@ -22,6 +23,8 @@ STRUCT_FIELD=Value STRUCTPTR_FIELD="Val\"ue " MONGO_DATABASE_HOST="mongodb://user:pass==@host.tld:955/?ssl=true&replicaSet=globaldb" # db connection MONGO_DATABASE_COLLECTION_NAME='us=ers' -MONGO_OTHER=1 +MONGO_OTHER=$A MONGO_X=97 # comment +INTERPOLATED="\$B env_$A $ \$B \\$C" + diff --git a/vars.go b/vars.go index 6432a4e..425f938 100644 --- a/vars.go +++ b/vars.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "strings" "unicode" ) @@ -93,3 +94,72 @@ func parseVars(r io.Reader, vars map[string]string) error { func varValue(v []byte) string { return string(bytes.Trim(bytes.TrimSpace(v), `"'`)) } + +func interpolateVars(vars map[string]string) { + for k, v := range vars { + if strings.IndexByte(v, '$') == -1 { + continue + } + + atVar := false + var name []byte + var newValue []byte + + for i := 0; i < len(v); i++ { + // Variable starts + if v[i] == '$' { + atVar = true + + // Variable is escaped + if i-1 >= 0 && v[i-1] == '\\' { + atVar = false + } + + // Variable is double escaped + if i-2 > 0 && v[i-2] == '\\' { + atVar = true + } + + if i+1 < len(v) && (unicode.IsSpace(rune(v[i+1])) || v[i+1] == '"' || v[i+1] == '\'') { + atVar = false + } + + if atVar { + continue + } + } + + if !atVar { + // Next variable is double escaped + if v[i] == '\\' && i+1 < len(v) && v[i+1] == '\\' && i+2 < len(v) && v[i+2] == '$' { + newValue = append(newValue, v[i]) + continue + } + + // Next variable is escaped + if v[i] == '\\' && i+1 < len(v) && v[i+1] == '$' { + continue + } + + newValue = append(newValue, v[i]) + continue + } + + if unicode.IsSpace(rune(v[i])) { + newValue = append(newValue, []byte(vars[string(name)])...) + newValue = append(newValue, v[i]) + name = nil + atVar = false + continue + } + + name = append(name, v[i]) + } + + if atVar { + newValue = append(newValue, []byte(vars[string(name)])...) + } + + vars[k] = string(newValue) + } +} From 437f1cf270ffa8b8b59b94210baf4eac9380f8ce Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Tue, 10 Dec 2019 21:04:52 +0200 Subject: [PATCH 5/6] Support more cases. --- .golangci.yml | 2 +- config_test.go | 4 ++-- testdata/.env | 2 +- vars.go | 40 +++++++++++++++++++++++++++------------- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0d34a53..438cb7d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -42,7 +42,7 @@ linters-settings: min-confidence: 0.8 gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 25 + min-complexity: 23 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/config_test.go b/config_test.go index 48a03fa..65948d7 100644 --- a/config_test.go +++ b/config_test.go @@ -185,7 +185,7 @@ func TestJson(t *testing.T) { "Struct":{ "Field":"Value" }, - "Interpolated":"$B env_1 $ $B \\3" + "Interpolated":"$B env_1 $ $B \\3 6379 + $" }`) _, expected, err := testdata() @@ -371,7 +371,7 @@ func testdata() ([]byte, Config, error) { X: 'a', }, }}, - Interpolated: "$B env_1 $ $B \\3", + Interpolated: "$B env_1 $ $B \\3 6379 + $", } return input, expected, nil diff --git a/testdata/.env b/testdata/.env index d9e927a..9ea8198 100644 --- a/testdata/.env +++ b/testdata/.env @@ -26,5 +26,5 @@ MONGO_DATABASE_COLLECTION_NAME='us=ers' MONGO_OTHER=$A MONGO_X=97 # comment -INTERPOLATED="\$B env_$A $ \$B \\$C" +INTERPOLATED="\$B env_$A $ \$B \\$C ${REDIS_PORT} + $" diff --git a/vars.go b/vars.go index 425f938..20e60ad 100644 --- a/vars.go +++ b/vars.go @@ -108,20 +108,10 @@ func interpolateVars(vars map[string]string) { for i := 0; i < len(v); i++ { // Variable starts if v[i] == '$' { - atVar = true + atVar = isAtVar(v, i) - // Variable is escaped - if i-1 >= 0 && v[i-1] == '\\' { - atVar = false - } - - // Variable is double escaped - if i-2 > 0 && v[i-2] == '\\' { - atVar = true - } - - if i+1 < len(v) && (unicode.IsSpace(rune(v[i+1])) || v[i+1] == '"' || v[i+1] == '\'') { - atVar = false + if i == len(v)-1 && i-1 >= 0 && v[i-1] != '\\' { + newValue = append(newValue, v[i]) } if atVar { @@ -145,6 +135,10 @@ func interpolateVars(vars map[string]string) { continue } + if atVar && (v[i] == '{' || v[i] == '}') { + continue + } + if unicode.IsSpace(rune(v[i])) { newValue = append(newValue, []byte(vars[string(name)])...) newValue = append(newValue, v[i]) @@ -163,3 +157,23 @@ func interpolateVars(vars map[string]string) { vars[k] = string(newValue) } } + +func isAtVar(v string, i int) bool { + atVar := true + + // Variable is escaped + if i-1 >= 0 && v[i-1] == '\\' { + atVar = false + } + + // Variable is double escaped + if i-2 > 0 && v[i-2] == '\\' { + atVar = true + } + + if i+1 < len(v) && (unicode.IsSpace(rune(v[i+1])) || v[i+1] == '"' || v[i+1] == '\'') { + atVar = false + } + + return atVar +} From b1941e95d0a399636d5a852f0b7b6a755f51d4d6 Mon Sep 17 00:00:00 2001 From: Andrei Avram Date: Tue, 10 Dec 2019 21:13:09 +0200 Subject: [PATCH 6/6] Extract cases. --- .golangci.yml | 2 +- vars.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 438cb7d..7985e2d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -42,7 +42,7 @@ linters-settings: min-confidence: 0.8 gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 23 + min-complexity: 17 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/vars.go b/vars.go index 20e60ad..c50850c 100644 --- a/vars.go +++ b/vars.go @@ -120,14 +120,12 @@ func interpolateVars(vars map[string]string) { } if !atVar { - // Next variable is double escaped - if v[i] == '\\' && i+1 < len(v) && v[i+1] == '\\' && i+2 < len(v) && v[i+2] == '$' { + if nextVarIsDoubleEscaped(v, i) { newValue = append(newValue, v[i]) continue } - // Next variable is escaped - if v[i] == '\\' && i+1 < len(v) && v[i+1] == '$' { + if nextVarIsEscaped(v, i) { continue } @@ -177,3 +175,11 @@ func isAtVar(v string, i int) bool { return atVar } + +func nextVarIsDoubleEscaped(v string, i int) bool { + return v[i] == '\\' && i+1 < len(v) && v[i+1] == '\\' && i+2 < len(v) && v[i+2] == '$' +} + +func nextVarIsEscaped(v string, i int) bool { + return v[i] == '\\' && i+1 < len(v) && v[i+1] == '$' +}