From 9eb65767e58a3a51a879b015b71110da332bdf2f Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Fri, 12 Jan 2024 17:02:57 -0500 Subject: [PATCH 1/2] render templates with external functions Some consumers of `consul-template` use it like a library, where the application runs the runner in-process. For projects like Nomad which need to run with a high level of privilege, this is problematic in that its challenging to secure the operations that read and write from disk without running the entirety of CT as an external process (which carries a lot of overhead for Nomad workloads). Add a `RendererFunc` and `ReaderFunc` interface to allow Nomad to inject a sandboxed subprocess when reading from disk and writing to disk. This implementation is currently being used in Nomad 1.7.4, 1.6.7, and 1.5.14 as a mitigation for https://github.com/hashicorp/nomad/issues/19888. See https://github.com/hashicorp/nomad/commit/df865033494574de14da687c999d66b799899b21 for example usage. --- cli_test.go | 10 ++++++++++ config/config.go | 27 +++++++++++++++++++++++++++ config/config_test.go | 6 ++++++ manager/runner.go | 16 +++++++++++++++- renderer/renderer.go | 3 +++ template/template.go | 6 ++++-- template/template_test.go | 3 +++ 7 files changed, 68 insertions(+), 3 deletions(-) diff --git a/cli_test.go b/cli_test.go index f0365785a..f40d9a8dd 100644 --- a/cli_test.go +++ b/cli_test.go @@ -909,6 +909,16 @@ func TestCLI_ParseFlags(t *testing.T) { e.Finalize() } + // these can't be compared with DeepEqual + if e != nil { + e.RendererFunc = nil + e.ReaderFunc = nil + } + if a != nil { + a.RendererFunc = nil + a.ReaderFunc = nil + } + if !reflect.DeepEqual(e, a) { t.Errorf("Config diff: %soutput: %q", e.Diff(a), out) } diff --git a/config/config.go b/config/config.go index 97ff3a0a4..627a9ecf9 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/hashicorp/consul-template/renderer" "github.com/hashicorp/consul-template/signals" "github.com/hashicorp/hcl" homedir "github.com/mitchellh/go-homedir" @@ -111,8 +112,17 @@ type Config struct { // ErrOnFailedLookup, when enabled, will trigger an error if a dependency // fails to return a value. ErrOnFailedLookup bool `mapstructure:"err_on_failed_lookup"` + + // RendererFunc is called whenever the template needs to be written, and + // will default to renderer.Render. This is intended for use when embedding + // Consul Template in another application + RendererFunc renderer.Renderer `mapstructure:"-" json:"-"` + + ReaderFunc Reader `mapstructure:"-" json:"-"` } +type Reader func(src string) ([]byte, error) + // Copy returns a deep copy of the current configuration. This is useful because // the nested data structures may be shared. func (c *Config) Copy() *Config { @@ -182,6 +192,9 @@ func (c *Config) Copy() *Config { o.Nomad = c.Nomad.Copy() } + o.RendererFunc = c.RendererFunc + o.ReaderFunc = c.ReaderFunc + return &o } @@ -275,6 +288,13 @@ func (c *Config) Merge(o *Config) *Config { r.Nomad = r.Nomad.Merge(o.Nomad) } + if o.RendererFunc != nil { + r.RendererFunc = o.RendererFunc + } + if o.ReaderFunc != nil { + r.ReaderFunc = o.ReaderFunc + } + return r } @@ -636,6 +656,13 @@ func (c *Config) Finalize() { if c.BlockQueryWaitTime == nil { c.BlockQueryWaitTime = TimeDuration(DefaultBlockQueryWaitTime) } + + if c.RendererFunc == nil { + c.RendererFunc = renderer.Render + } + if c.ReaderFunc == nil { + c.ReaderFunc = os.ReadFile + } } func stringFromEnv(list []string, def string) *string { diff --git a/config/config_test.go b/config/config_test.go index 948909749..ceeee8f66 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2616,6 +2616,12 @@ func TestDefaultConfig(t *testing.T) { c := DefaultConfig() c.Finalize() + // these can't be compared with DeepEqual + c.RendererFunc = nil + c.ReaderFunc = nil + r.RendererFunc = nil + r.ReaderFunc = nil + if !reflect.DeepEqual(r, c) { t.Errorf("Config diff: %s", r.Diff(c)) } diff --git a/manager/runner.go b/manager/runner.go index 72128d979..a884a273a 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -114,6 +114,10 @@ type Runner struct { // stopped is a boolean of whether the runner is stopped stopped bool + rendererFn renderer.Renderer + + readerFn config.Reader + // finalConfigCopy provides access to a static copy of the finalized // Runner config. This prevents risk of data races when reading config for // other elements started by the Runner, like template functions. @@ -200,6 +204,15 @@ func NewRunner(config *config.Config, dry bool) (*Runner, error) { brain: template.NewBrain(), quiescenceMap: make(map[string]*quiescence), quiescenceCh: make(chan *template.Template), + rendererFn: config.RendererFunc, + readerFn: config.ReaderFunc, + } + + if runner.rendererFn == nil { + runner.rendererFn = renderer.Render + } + if runner.readerFn == nil { + runner.readerFn = os.ReadFile } // Create the clientset @@ -853,7 +866,7 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (* log.Printf("[DEBUG] (runner) rendering %s", templateConfig.Display()) // Render the template, taking dry mode into account - result, err := renderer.Render(&renderer.RenderInput{ + result, err := r.rendererFn(&renderer.RenderInput{ Backup: config.BoolVal(templateConfig.Backup), Contents: result.Output, CreateDestDirs: config.BoolVal(templateConfig.CreateDestDirs), @@ -975,6 +988,7 @@ func (r *Runner) init(clients *dep.ClientSet) error { SandboxPath: config.StringVal(ctmpl.SandboxPath), Destination: config.StringVal(ctmpl.Destination), Config: ctmpl, + ReaderFunc: r.config.ReaderFunc, }) if err != nil { return err diff --git a/renderer/renderer.go b/renderer/renderer.go index c8fd2dfc8..ccc5fc1a8 100644 --- a/renderer/renderer.go +++ b/renderer/renderer.go @@ -59,9 +59,12 @@ type RenderResult struct { Contents []byte } +type Renderer func(*RenderInput) (*RenderResult, error) + // Render atomically renders a file contents to disk, returning a result of // whether it would have rendered and actually did render. func Render(i *RenderInput) (*RenderResult, error) { + existing, err := os.ReadFile(i.Path) fileExists := !os.IsNotExist(err) if err != nil && fileExists { diff --git a/template/template.go b/template/template.go index ffc624c2e..7beb4d729 100644 --- a/template/template.go +++ b/template/template.go @@ -8,7 +8,6 @@ import ( "crypto/md5" "encoding/hex" "fmt" - "os" "strings" "text/template" @@ -117,6 +116,9 @@ type NewTemplateInput struct { // Config keeps local reference to config struct Config *config.TemplateConfig + + // ReaderFunc is called to read in any source file + ReaderFunc config.Reader } // NewTemplate creates and parses a new Consul Template template at the given @@ -154,7 +156,7 @@ func NewTemplate(i *NewTemplateInput) (*Template, error) { } if i.Source != "" { - contents, err := os.ReadFile(i.Source) + contents, err := i.ReaderFunc(i.Source) if err != nil { return nil, errors.Wrap(err, "failed to read template") } diff --git a/template/template_test.go b/template/template_test.go index 7b21a8318..a9def4d7a 100644 --- a/template/template_test.go +++ b/template/template_test.go @@ -119,6 +119,9 @@ func TestNewTemplate(t *testing.T) { for i, tc := range cases { t.Run(fmt.Sprintf("%d_%s", i, tc.name), func(t *testing.T) { + if tc.i != nil { + tc.i.ReaderFunc = os.ReadFile + } a, err := NewTemplate(tc.i) if (err != nil) != tc.err { t.Fatal(err) From d0656c4561ba98642e5ed7cf47b4001f491b5cd7 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Mon, 4 Mar 2024 16:06:41 -0500 Subject: [PATCH 2/2] address comments from code review --- config/config.go | 7 +++++++ manager/runner.go | 6 ++++++ renderer/renderer.go | 1 - template/template.go | 7 +++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 627a9ecf9..c8109a7da 100644 --- a/config/config.go +++ b/config/config.go @@ -118,9 +118,16 @@ type Config struct { // Consul Template in another application RendererFunc renderer.Renderer `mapstructure:"-" json:"-"` + // ReaderFunc is called whenever the template source is read, and will + // default to os.ReadFile. This is intended for use when embedding Consul + // Template in another application. ReaderFunc Reader `mapstructure:"-" json:"-"` } +// Reader is an interface that is implemented by os.OpenFile. The +// Config.ReaderFunc requires this interface so that applications that embed +// Consul Template can have an alternative implementation of os.OpenFile +// (ex. virtual file, sandboxed reads) type Reader func(src string) ([]byte, error) // Copy returns a deep copy of the current configuration. This is useful because diff --git a/manager/runner.go b/manager/runner.go index a884a273a..856822b4e 100644 --- a/manager/runner.go +++ b/manager/runner.go @@ -114,8 +114,14 @@ type Runner struct { // stopped is a boolean of whether the runner is stopped stopped bool + // rendererFn is called whenever the template needs to be written, and will + // default to renderer.Render. This is intended for use when embedding + // Consul Template in another application rendererFn renderer.Renderer + // readerFn is called whenever the template source is read, and will default + // to os.ReadFile. This is intended for use when embedding Consul Template + // in another application. readerFn config.Reader // finalConfigCopy provides access to a static copy of the finalized diff --git a/renderer/renderer.go b/renderer/renderer.go index ccc5fc1a8..5abe45ecf 100644 --- a/renderer/renderer.go +++ b/renderer/renderer.go @@ -64,7 +64,6 @@ type Renderer func(*RenderInput) (*RenderResult, error) // Render atomically renders a file contents to disk, returning a result of // whether it would have rendered and actually did render. func Render(i *RenderInput) (*RenderResult, error) { - existing, err := os.ReadFile(i.Path) fileExists := !os.IsNotExist(err) if err != nil && fileExists { diff --git a/template/template.go b/template/template.go index 7beb4d729..28b5265d1 100644 --- a/template/template.go +++ b/template/template.go @@ -27,6 +27,10 @@ var ( // does not specify either a "source" or "content" argument, which is not // valid. ErrTemplateMissingContentsAndSource = errors.New("template: must specify exactly one of 'source' or 'contents'") + + // ErrMissingReaderFunction is the error returned when the template + // configuration is missing a reader function. + ErrMissingReaderFunction = errors.New("template: missing a reader function") ) // Template is the internal representation of an individual template to process. @@ -156,6 +160,9 @@ func NewTemplate(i *NewTemplateInput) (*Template, error) { } if i.Source != "" { + if i.ReaderFunc == nil { + return nil, ErrMissingReaderFunction + } contents, err := i.ReaderFunc(i.Source) if err != nil { return nil, errors.Wrap(err, "failed to read template")