diff --git a/cli/app.go b/cli/app.go index 10f2e77bb..92dc2118a 100644 --- a/cli/app.go +++ b/cli/app.go @@ -127,9 +127,11 @@ func (app *App) RunContext(ctx context.Context, args []string) error { } }(ctx) + ctx = config.WithConfigValues(ctx) + // init engine if required if engine.IsEngineEnabled() { - ctx = engine.ContextWithEngine(ctx) + ctx = engine.WithEngineValues(ctx) } defer func(ctx context.Context) { if err := engine.Shutdown(ctx); err != nil { diff --git a/cli/commands/terraform/creds/providers/amazonsts/provider.go b/cli/commands/terraform/creds/providers/amazonsts/provider.go index 8452f6ffa..9bf56798e 100644 --- a/cli/commands/terraform/creds/providers/amazonsts/provider.go +++ b/cli/commands/terraform/creds/providers/amazonsts/provider.go @@ -35,7 +35,7 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden return nil, nil } - if cached, hit := credentialsCache.Get(iamRoleOpts.RoleARN); hit { + if cached, hit := credentialsCache.Get(ctx, iamRoleOpts.RoleARN); hit { provider.terragruntOptions.Logger.Debugf("Using cached credentials for IAM role %s.", iamRoleOpts.RoleARN) return cached, nil } @@ -56,10 +56,10 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden }, } - credentialsCache.Put(iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second)) + credentialsCache.Put(ctx, iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second)) return creds, nil } // credentialsCache is a cache of credentials. -var credentialsCache = cache.NewExpiringCache[*providers.Credentials]() +var credentialsCache = cache.NewExpiringCache[*providers.Credentials]("credentialsCache") diff --git a/cli/commands/terraform/version_check.go b/cli/commands/terraform/version_check.go index 680cd0fde..eb5b91e50 100644 --- a/cli/commands/terraform/version_check.go +++ b/cli/commands/terraform/version_check.go @@ -34,7 +34,7 @@ const versionParts = 3 // - TerraformVersion // TODO: Look into a way to refactor this function to avoid the side effect. func checkVersionConstraints(ctx context.Context, terragruntOptions *options.TerragruntOptions) error { - configContext := config.NewParsingContext(context.Background(), terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints) + configContext := config.NewParsingContext(ctx, terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints) partialTerragruntConfig, err := config.PartialParseConfigFile( configContext, diff --git a/config/cache_test.go b/config/cache_test.go index beedfff48..c567173e4 100644 --- a/config/cache_test.go +++ b/config/cache_test.go @@ -1,16 +1,19 @@ package config import ( + "context" "testing" "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/stretchr/testify/assert" ) +const testCacheName = "TerragruntConfig" + func TestTerragruntConfigCacheCreation(t *testing.T) { t.Parallel() - cache := cache.NewCache[TerragruntConfig]() + cache := cache.NewCache[TerragruntConfig](testCacheName) assert.NotNil(t, cache.Mutex) assert.NotNil(t, cache.Cache) @@ -23,9 +26,10 @@ func TestTerragruntConfigCacheOperation(t *testing.T) { testCacheKey := "super-safe-cache-key" - cache := cache.NewCache[TerragruntConfig]() + ctx := context.Background() + cache := cache.NewCache[TerragruntConfig](testCacheName) - actualResult, found := cache.Get(testCacheKey) + actualResult, found := cache.Get(ctx, testCacheKey) assert.False(t, found) assert.Empty(t, actualResult) @@ -34,8 +38,8 @@ func TestTerragruntConfigCacheOperation(t *testing.T) { IsPartial: true, // Any random property will be sufficient } - cache.Put(testCacheKey, stubTerragruntConfig) - actualResult, found = cache.Get(testCacheKey) + cache.Put(ctx, testCacheKey, stubTerragruntConfig) + actualResult, found = cache.Get(ctx, testCacheKey) assert.True(t, found) assert.NotEmpty(t, actualResult) diff --git a/config/config.go b/config/config.go index fb1a43f68..d6a999b1d 100644 --- a/config/config.go +++ b/config/config.go @@ -33,13 +33,11 @@ import ( const ( DefaultTerragruntConfigPath = "terragrunt.hcl" DefaultTerragruntJsonConfigPath = "terragrunt.hcl.json" + FoundInFile = "found_in_file" - DefaultEngineType = "rpc" -) - -const FoundInFile = "found_in_file" + iamRoleCacheName = "iamRoleCache" -const ( + DefaultEngineType = "rpc" MetadataTerraform = "terraform" MetadataTerraformBinary = "terraform_binary" MetadataTerraformVersionConstraint = "terraform_version_constraint" @@ -696,12 +694,11 @@ func ReadTerragruntConfig(ctx context.Context, terragruntOptions *options.Terrag return ParseConfigFile(parcingCtx, terragruntOptions.TerragruntConfigPath, nil) } -var hclCache = cache.NewCache[*hclparse.File]() - // Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config // included in some other config file when resolving relative paths. func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { var config *TerragruntConfig + hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey) err := telemetry.Telemetry(ctx, ctx.TerragruntOptions, "parse_config_file", map[string]interface{}{ "config_path": configPath, "working_dir": ctx.TerragruntOptions.WorkingDir, @@ -725,7 +722,7 @@ func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *I } var file *hclparse.File var cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, ctx.TerragruntOptions.WorkingDir, dir, fileInfo.ModTime().UnixMicro()) - if cacheConfig, found := hclCache.Get(cacheKey); found { + if cacheConfig, found := hclCache.Get(ctx, cacheKey); found { file = cacheConfig } else { // Parse the HCL file into an AST body that can be decoded multiple times later without having to re-parse @@ -733,7 +730,7 @@ func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *I if err != nil { return err } - hclCache.Put(cacheKey, file) + hclCache.Put(ctx, cacheKey, file) } config, err = ParseConfig(ctx, file, includeFromChild) if err != nil { @@ -856,7 +853,7 @@ func ParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *Inc } // iamRoleCache - store for cached values of IAM roles -var iamRoleCache = cache.NewCache[options.IAMRoleOptions]() +var iamRoleCache = cache.NewCache[options.IAMRoleOptions](iamRoleCacheName) // setIAMRole - extract IAM role details from Terragrunt flags block func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) error { @@ -866,14 +863,14 @@ func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *Incl } else { // as key is considered HCL code and include configuration var key = fmt.Sprintf("%v-%v", file.Content(), includeFromChild) - var config, found = iamRoleCache.Get(key) + var config, found = iamRoleCache.Get(ctx, key) if !found { iamConfig, err := TerragruntConfigFromPartialConfig(ctx.WithDecodeList(TerragruntFlags), file, includeFromChild) if err != nil { return err } config = iamConfig.GetIAMRoleOptions() - iamRoleCache.Put(key, config) + iamRoleCache.Put(ctx, key, config) } // We merge the OriginalIAMRoleOptions into the one from the config, because the CLI passed IAMRoleOptions has // precedence. diff --git a/config/config_helpers.go b/config/config_helpers.go index da8894b6d..95afd460a 100644 --- a/config/config_helpers.go +++ b/config/config_helpers.go @@ -69,6 +69,8 @@ const ( FuncNameEndsWith = "endswith" FuncNameStrContains = "strcontains" FuncNameTimeCmp = "timecmp" + + sopsCacheName = "sopsCache" ) // List of terraform commands that accept -lock-timeout @@ -308,14 +310,14 @@ func parseGetEnvParameters(parameters []string) (EnvVar, error) { return envVariable, nil } -// runCommandCache - cache of evaluated `run_cmd` invocations -// see: https://github.com/gruntwork-io/terragrunt/issues/1427 -var runCommandCache = cache.NewCache[string]() - // runCommand is a helper function that runs a command and returns the stdout as the interporation // for each `run_cmd` in locals section, function is called twice // result func runCommand(ctx *ParsingContext, args []string) (string, error) { + // runCommandCache - cache of evaluated `run_cmd` invocations + // see: https://github.com/gruntwork-io/terragrunt/issues/1427 + runCommandCache := cache.ContextCache[string](ctx, RunCmdCacheContextKey) + if len(args) == 0 { return "", errors.WithStackTrace(EmptyStringNotAllowedError("parameter to the run_cmd function")) } @@ -341,7 +343,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) { // To avoid re-run of the same run_cmd command, is used in memory cache for command results, with caching key path + arguments // see: https://github.com/gruntwork-io/terragrunt/issues/1427 cacheKey := fmt.Sprintf("%v-%v", cachePath, args) - cachedValue, foundInCache := runCommandCache.Get(cacheKey) + cachedValue, foundInCache := runCommandCache.Get(ctx, cacheKey) if foundInCache { if suppressOutput { ctx.TerragruntOptions.Logger.Debugf("run_cmd, cached output: [REDACTED]") @@ -366,7 +368,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) { // Persisting result in cache to avoid future re-evaluation // see: https://github.com/gruntwork-io/terragrunt/issues/1427 - runCommandCache.Put(cacheKey, value) + runCommandCache.Put(ctx, cacheKey, value) return value, nil } @@ -728,7 +730,7 @@ func getModulePathFromSourceUrl(sourceUrl string) (string, error) { // // The cache keys are the canonical paths to the encrypted files, and the values are the // plain-text result of the decrypt operation. -var sopsCache = cache.NewCache[string]() +var sopsCache = cache.NewCache[string](sopsCacheName) // decrypts and returns sops encrypted utf-8 yaml or json data as a string func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) { @@ -751,7 +753,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) { return "", errors.WithStackTrace(err) } - if val, ok := sopsCache.Get(canonicalSourceFile); ok { + if val, ok := sopsCache.Get(ctx, canonicalSourceFile); ok { return val, nil } @@ -762,7 +764,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) { if utf8.Valid(rawData) { value := string(rawData) - sopsCache.Put(canonicalSourceFile, value) + sopsCache.Put(ctx, canonicalSourceFile, value) return value, nil } diff --git a/config/config_partial.go b/config/config_partial.go index 93db34e27..e6fa0528b 100644 --- a/config/config_partial.go +++ b/config/config_partial.go @@ -2,14 +2,18 @@ package config import ( "fmt" + "os" "path/filepath" + clone "github.com/huandu/go-clone" + + "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" "github.com/gruntwork-io/go-commons/errors" "github.com/gruntwork-io/terragrunt/config/hclparse" - "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/util" ) @@ -136,26 +140,39 @@ func DecodeBaseBlocks(ctx *ParsingContext, file *hclparse.File, includeFromChild } func PartialParseConfigFile(ctx *ParsingContext, configPath string, include *IncludeConfig) (*TerragruntConfig, error) { - file, err := hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath) + hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey) + + fileInfo, err := os.Stat(configPath) if err != nil { return nil, err } + var file *hclparse.File + var cacheKey = fmt.Sprintf("configPath-%v-modTime-%v", configPath, fileInfo.ModTime().UnixMicro()) + + if cacheConfig, found := hclCache.Get(ctx, cacheKey); found { + file = cacheConfig + } else { + file, err = hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath) + if err != nil { + return nil, err + } + } return TerragruntConfigFromPartialConfig(ctx, file, include) } -var terragruntConfigCache = cache.NewCache[TerragruntConfig]() - // Wrapper of PartialParseConfigString which checks for cached configs. // filename, configString, includeFromChild and decodeList are used for the cache key, // by getting the default value (%#v) through fmt. func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) { var cacheKey = fmt.Sprintf("%#v-%#v-%#v-%#v", file.ConfigPath, file.Content(), includeFromChild, ctx.PartialParseDecodeList) + terragruntConfigCache := cache.ContextCache[*TerragruntConfig](ctx, RunCmdCacheContextKey) if ctx.TerragruntOptions.UsePartialParseConfigCache { - if config, found := terragruntConfigCache.Get(cacheKey); found { + if config, found := terragruntConfigCache.Get(ctx, cacheKey); found { ctx.TerragruntOptions.Logger.Debugf("Cache hit for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList) - return &config, nil + deepCopy := clone.Clone(config).(*TerragruntConfig) + return deepCopy, nil } ctx.TerragruntOptions.Logger.Debugf("Cache miss for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList) @@ -167,7 +184,8 @@ func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File, } if ctx.TerragruntOptions.UsePartialParseConfigCache { - terragruntConfigCache.Put(cacheKey, *config) + putConfig := clone.Clone(config).(*TerragruntConfig) + terragruntConfigCache.Put(ctx, cacheKey, putConfig) } return config, nil diff --git a/config/context.go b/config/context.go new file mode 100644 index 000000000..d4cf5ecab --- /dev/null +++ b/config/context.go @@ -0,0 +1,31 @@ +package config + +import ( + "context" + + "github.com/gruntwork-io/terragrunt/config/hclparse" + "github.com/gruntwork-io/terragrunt/internal/cache" +) + +type configKey byte + +const ( + HclCacheContextKey configKey = iota + TerragruntConfigCacheContextKey configKey = iota + RunCmdCacheContextKey configKey = iota + DependencyOutputCacheContextKey configKey = iota + + hclCacheName = "hclCache" + configCacheName = "configCache" + runCmdCacheName = "runCmdCache" + dependencyOutputCacheName = "dependencyOutputCache" +) + +// WithConfigValues add to context default values for configuration. +func WithConfigValues(ctx context.Context) context.Context { + ctx = context.WithValue(ctx, HclCacheContextKey, cache.NewCache[*hclparse.File](hclCacheName)) + ctx = context.WithValue(ctx, TerragruntConfigCacheContextKey, cache.NewCache[*TerragruntConfig](configCacheName)) + ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName)) + ctx = context.WithValue(ctx, DependencyOutputCacheContextKey, cache.NewCache[*dependencyOutputCache](dependencyOutputCacheName)) + return ctx +} diff --git a/config/dependency.go b/config/dependency.go index fd90d6962..fe931c096 100644 --- a/config/dependency.go +++ b/config/dependency.go @@ -11,6 +11,8 @@ import ( "strings" "sync" + "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/hashicorp/go-getter" @@ -37,16 +39,10 @@ const renderJsonCommand = "render-json" type Dependencies []Dependency -func (deps Dependencies) FilteredWithoutConfigPath() Dependencies { - var filteredDeps Dependencies - - for _, dep := range deps { - if !dep.ConfigPath.IsNull() { - filteredDeps = append(filteredDeps, dep) - } - } - - return filteredDeps +// Struct to hold the decoded dependency blocks. +type dependencyOutputCache struct { + Enabled *bool + Inputs cty.Value } type Dependency struct { @@ -200,34 +196,11 @@ func decodeAndRetrieveOutputs(ctx *ParsingContext, file *hclparse.File) (*cty.Va return nil, err } - // Mark skipped dependencies as disabled - updatedDependencies := terragruntDependency{} - for _, dep := range decodedDependency.Dependencies { - depPath := getCleanedTargetConfigPath(dep.ConfigPath.AsString(), ctx.TerragruntOptions.TerragruntConfigPath) - if dep.isEnabled() && util.FileExists(depPath) { - depOpts := cloneTerragruntOptionsForDependency(ctx, depPath) - depCtx := ctx.WithDecodeList(TerragruntFlags, TerragruntInputs).WithTerragruntOptions(depOpts) - - if depConfig, err := PartialParseConfigFile(depCtx, depPath, nil); err == nil { - if depConfig.Skip { - ctx.TerragruntOptions.Logger.Debugf("Skipping outputs reading for disabled dependency %s", dep.Name) - dep.Enabled = new(bool) - } - - inputsCty, err := convertToCtyWithJson(depConfig.Inputs) - if err != nil { - return nil, err - } - dep.Inputs = &inputsCty - - } else { - ctx.TerragruntOptions.Logger.Warnf("Error reading partial config for dependency %s: %v", dep.Name, err) - } - } - - updatedDependencies.Dependencies = append(updatedDependencies.Dependencies, dep) + updatedDependencies, err := decodeDependencies(ctx, decodedDependency) + if err != nil { + return nil, err } - decodedDependency = updatedDependencies + decodedDependency = *updatedDependencies // Merge in included dependencies if ctx.TrackInclude != nil { @@ -241,6 +214,46 @@ func decodeAndRetrieveOutputs(ctx *ParsingContext, file *hclparse.File) (*cty.Va return dependencyBlocksToCtyValue(ctx, decodedDependency.Dependencies) } +// decodeDependencies decode dependencies and fetch inputs +func decodeDependencies(ctx *ParsingContext, decodedDependency terragruntDependency) (*terragruntDependency, error) { + updatedDependencies := terragruntDependency{} + depCache := cache.ContextCache[*dependencyOutputCache](ctx, DependencyOutputCacheContextKey) + for _, dep := range decodedDependency.Dependencies { + depPath := getCleanedTargetConfigPath(dep.ConfigPath.AsString(), ctx.TerragruntOptions.TerragruntConfigPath) + if dep.isEnabled() && util.FileExists(depPath) { + cacheKey := ctx.TerragruntOptions.WorkingDir + depPath + cachedDependency, found := depCache.Get(ctx, cacheKey) + if !found { + depOpts := cloneTerragruntOptionsForDependency(ctx, depPath) + depCtx := ctx.WithDecodeList(TerragruntFlags, TerragruntInputs).WithTerragruntOptions(depOpts) + if depConfig, err := PartialParseConfigFile(depCtx, depPath, nil); err == nil { + if depConfig.Skip { + ctx.TerragruntOptions.Logger.Debugf("Skipping outputs reading for disabled dependency %s", dep.Name) + dep.Enabled = new(bool) + } + inputsCty, err := convertToCtyWithJson(depConfig.Inputs) + if err != nil { + return nil, err + } + cachedValue := dependencyOutputCache{ + Enabled: dep.Enabled, + Inputs: inputsCty, + } + depCache.Put(ctx, cacheKey, &cachedValue) + dep.Inputs = &inputsCty + } else { + ctx.TerragruntOptions.Logger.Warnf("Error reading partial config for dependency %s: %v", dep.Name, err) + } + } else { + dep.Enabled = cachedDependency.Enabled + dep.Inputs = &cachedDependency.Inputs + } + } + updatedDependencies.Dependencies = append(updatedDependencies.Dependencies, dep) + } + return &updatedDependencies, nil +} + // Convert the list of parsed Dependency blocks into a list of module dependencies. Each output block should // become a dependency of the current config, since that module has to be applied before we can read the output. func dependencyBlocksToModuleDependencies(decodedDependencyBlocks []Dependency) *ModuleDependencies { @@ -999,3 +1012,15 @@ func runTerraformInitForDependencyOutput(ctx *ParsingContext, workingDir string, ctx.TerragruntOptions.Logger.Debugf(stderr.String()) } } + +func (deps Dependencies) FilteredWithoutConfigPath() Dependencies { + var filteredDeps Dependencies + + for _, dep := range deps { + if !dep.ConfigPath.IsNull() { + filteredDeps = append(filteredDeps, dep) + } + } + + return filteredDeps +} diff --git a/configstack/module.go b/configstack/module.go index 755ef5d21..16d78ea02 100644 --- a/configstack/module.go +++ b/configstack/module.go @@ -23,6 +23,7 @@ import ( ) const maxLevelsOfRecursion = 20 +const existingModulesCacheName = "existingModules" // Represents a single module (i.e. folder with Terraform templates), including the Terragrunt configuration for that // module and the list of other modules that this module depends on @@ -491,7 +492,7 @@ func (modules TerraformModules) flagModulesThatDontInclude(terragruntOptions *op return modules, nil } -var existingModules = cache.NewCache[*TerraformModulesMap]() +var existingModules = cache.NewCache[*TerraformModulesMap](existingModulesCacheName) type TerraformModulesMap map[string]*TerraformModule diff --git a/configstack/stack.go b/configstack/stack.go index 8a50e486e..ab2a81e08 100644 --- a/configstack/stack.go +++ b/configstack/stack.go @@ -564,7 +564,7 @@ func (stack *Stack) resolveDependenciesForModule(ctx context.Context, module *Te } key := fmt.Sprintf("%s-%s-%v-%v", module.Path, stack.terragruntOptions.WorkingDir, skipExternal, stack.terragruntOptions.TerraformCommand) - if value, ok := existingModules.Get(key); ok { + if value, ok := existingModules.Get(ctx, key); ok { return *value, nil } @@ -592,7 +592,7 @@ func (stack *Stack) resolveDependenciesForModule(ctx context.Context, module *Te return nil, err } - existingModules.Put(key, &result) + existingModules.Put(ctx, key, &result) return result, nil } diff --git a/engine/engine.go b/engine/engine.go index bdecdbe7b..f0281d550 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -113,7 +113,8 @@ func Run( return cmdOutput, nil } -func ContextWithEngine(ctx context.Context) context.Context { +// WithEngineValues add to context default values for engine. +func WithEngineValues(ctx context.Context) context.Context { if !IsEngineEnabled() { return ctx } @@ -122,6 +123,7 @@ func ContextWithEngine(ctx context.Context) context.Context { return ctx } +// DownloadEngine downloads the engine for the given options. func DownloadEngine(ctx context.Context, opts *options.TerragruntOptions) error { if !IsEngineEnabled() { return nil diff --git a/go.mod b/go.mod index 87c1c134d..804fc2e4c 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,8 @@ require ( github.com/hashicorp/go-getter/v2 v2.2.1 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.1 + github.com/hashicorp/terraform-svchost v0.0.1 + github.com/huandu/go-clone/generic v1.7.2 github.com/labstack/echo/v4 v4.11.4 github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db @@ -187,8 +189,8 @@ require ( github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/logutils v1.0.0 // indirect github.com/hashicorp/terraform-registry-address v0.2.0 // indirect - github.com/hashicorp/terraform-svchost v0.0.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/huandu/go-clone v1.6.0 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect diff --git a/go.sum b/go.sum index 1817313ed..8e0a8f2ab 100644 --- a/go.sum +++ b/go.sum @@ -773,6 +773,12 @@ github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbg github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc= +github.com/huandu/go-clone v1.6.0/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= +github.com/huandu/go-clone/generic v1.7.2 h1:47pQphxs1Xc9cVADjOHN+Bm5D0hNagwH9UXErbxgVKA= +github.com/huandu/go-clone/generic v1.7.2/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 7c749f2fc..f600b750f 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -1,40 +1,52 @@ package cache import ( + "context" "crypto/sha256" "fmt" "sync" "time" + + "github.com/gruntwork-io/terragrunt/telemetry" ) // Cache - generic cache implementation type Cache[V any] struct { + Name string Cache map[string]V - Mutex *sync.Mutex + Mutex *sync.RWMutex } // NewCache - create new cache with generic type V -func NewCache[V any]() *Cache[V] { +func NewCache[V any](name string) *Cache[V] { return &Cache[V]{ + Name: name, Cache: make(map[string]V), - Mutex: &sync.Mutex{}, + Mutex: &sync.RWMutex{}, } } // Get - fetch value from cache by key -func (c *Cache[V]) Get(key string) (V, bool) { - c.Mutex.Lock() - defer c.Mutex.Unlock() +func (c *Cache[V]) Get(ctx context.Context, key string) (V, bool) { + c.Mutex.RLock() + defer c.Mutex.RUnlock() keyHash := sha256.Sum256([]byte(key)) cacheKey := fmt.Sprintf("%x", keyHash) value, found := c.Cache[cacheKey] + telemetry.Count(ctx, fmt.Sprintf("%s_cache_get", c.Name), 1) + if found { + telemetry.Count(ctx, fmt.Sprintf("%s_cache_hit", c.Name), 1) + } else { + telemetry.Count(ctx, fmt.Sprintf("%s_cache_miss", c.Name), 1) + } return value, found } // Put - put value into cache by key -func (c *Cache[V]) Put(key string, value V) { +func (c *Cache[V]) Put(ctx context.Context, key string, value V) { c.Mutex.Lock() defer c.Mutex.Unlock() + telemetry.Count(ctx, fmt.Sprintf("%s_cache_put", c.Name), 1) keyHash := sha256.Sum256([]byte(key)) cacheKey := fmt.Sprintf("%x", keyHash) c.Cache[cacheKey] = value @@ -48,36 +60,53 @@ type ExpiringItem[V any] struct { // ExpiringCache - cache with items with expiration time type ExpiringCache[V any] struct { + Name string Cache map[string]ExpiringItem[V] - Mutex *sync.Mutex + Mutex *sync.RWMutex } // NewExpiringCache - create new cache with generic type V -func NewExpiringCache[V any]() *ExpiringCache[V] { +func NewExpiringCache[V any](name string) *ExpiringCache[V] { return &ExpiringCache[V]{ + Name: name, Cache: make(map[string]ExpiringItem[V]), - Mutex: &sync.Mutex{}, + Mutex: &sync.RWMutex{}, } } // Get - fetch value from cache by key -func (c *ExpiringCache[V]) Get(key string) (V, bool) { - c.Mutex.Lock() - defer c.Mutex.Unlock() +func (c *ExpiringCache[V]) Get(ctx context.Context, key string) (V, bool) { + c.Mutex.RLock() + defer c.Mutex.RUnlock() item, found := c.Cache[key] + telemetry.Count(ctx, fmt.Sprintf("%s_cache_get", c.Name), 1) if !found { + telemetry.Count(ctx, fmt.Sprintf("%s_cache_miss", c.Name), 1) return item.Value, false } if time.Now().After(item.Expiration) { + telemetry.Count(ctx, fmt.Sprintf("%s_cache_expiry", c.Name), 1) delete(c.Cache, key) return item.Value, false } + telemetry.Count(ctx, fmt.Sprintf("%s_cache_hit", c.Name), 1) return item.Value, true } // Put - put value into cache by key -func (c *ExpiringCache[V]) Put(key string, value V, expiration time.Time) { +func (c *ExpiringCache[V]) Put(ctx context.Context, key string, value V, expiration time.Time) { c.Mutex.Lock() defer c.Mutex.Unlock() + + telemetry.Count(ctx, fmt.Sprintf("%s_cache_put", c.Name), 1) c.Cache[key] = ExpiringItem[V]{Value: value, Expiration: expiration} } + +// ContextCache returns cache from the context. If the cache is nil, it creates a new instance. +func ContextCache[T any](ctx context.Context, key any) *Cache[T] { + cacheInstance, ok := ctx.Value(key).(*Cache[T]) + if !ok || cacheInstance == nil { + cacheInstance = NewCache[T](fmt.Sprintf("%v", key)) + } + return cacheInstance +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 587f46dc7..542fab1b4 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -1,6 +1,7 @@ package cache import ( + "context" "testing" "time" @@ -10,7 +11,7 @@ import ( func TestCacheCreation(t *testing.T) { t.Parallel() - cache := NewCache[string]() + cache := NewCache[string]("test") assert.NotNil(t, cache.Mutex) assert.NotNil(t, cache.Cache) @@ -21,15 +22,16 @@ func TestCacheCreation(t *testing.T) { func TestStringCacheOperation(t *testing.T) { t.Parallel() - cache := NewCache[string]() + ctx := context.Background() + cache := NewCache[string]("test") - value, found := cache.Get("potato") + value, found := cache.Get(ctx, "potato") assert.False(t, found) assert.Empty(t, value) - cache.Put("potato", "carrot") - value, found = cache.Get("potato") + cache.Put(ctx, "potato", "carrot") + value, found = cache.Get(ctx, "potato") assert.True(t, found) assert.NotEmpty(t, value) @@ -39,7 +41,7 @@ func TestStringCacheOperation(t *testing.T) { func TestExpiringCacheCreation(t *testing.T) { t.Parallel() - cache := NewExpiringCache[string]() + cache := NewExpiringCache[string]("test") assert.NotNil(t, cache.Mutex) assert.NotNil(t, cache.Cache) @@ -50,15 +52,16 @@ func TestExpiringCacheCreation(t *testing.T) { func TestExpiringCacheOperation(t *testing.T) { t.Parallel() - cache := NewExpiringCache[string]() + ctx := context.Background() + cache := NewExpiringCache[string]("test") - value, found := cache.Get("potato") + value, found := cache.Get(ctx, "potato") assert.False(t, found) assert.Empty(t, value) - cache.Put("potato", "carrot", time.Now().Add(1*time.Second)) - value, found = cache.Get("potato") + cache.Put(ctx, "potato", "carrot", time.Now().Add(1*time.Second)) + value, found = cache.Get(ctx, "potato") assert.True(t, found) assert.NotEmpty(t, value) @@ -68,10 +71,11 @@ func TestExpiringCacheOperation(t *testing.T) { func TestExpiringCacheExpiration(t *testing.T) { t.Parallel() - cache := NewExpiringCache[string]() + ctx := context.Background() + cache := NewExpiringCache[string]("test") - cache.Put("potato", "carrot", time.Now().Add(-1*time.Second)) - value, found := cache.Get("potato") + cache.Put(ctx, "potato", "carrot", time.Now().Add(-1*time.Second)) + value, found := cache.Get(ctx, "potato") assert.False(t, found) assert.NotEmpty(t, value) diff --git a/remote/remote_state.go b/remote/remote_state.go index f481a8e31..f3ffb7007 100644 --- a/remote/remote_state.go +++ b/remote/remote_state.go @@ -13,6 +13,8 @@ import ( "github.com/gruntwork-io/terragrunt/options" ) +const initializedRemoteStateCacheName = "initializedRemoteStateCache" + // Configuration for Terraform remote state // NOTE: If any attributes are added here, be sure to add it to remoteStateAsCty in config/config_as_cty.go type RemoteState struct { @@ -33,7 +35,7 @@ var stateAccessLock = newStateAccess() // initializedRemoteStateCache is a cache to store the result of a remote state initialization check. // This is used to avoid checking to see if remote state needs to be initialized multiple times. -var initializedRemoteStateCache = cache.NewCache[bool]() +var initializedRemoteStateCache = cache.NewCache[bool](initializedRemoteStateCacheName) func (remoteState *RemoteState) String() string { return fmt.Sprintf("RemoteState{Backend = %v, DisableInit = %v, DisableDependencyOptimization = %v, Generate = %v, Config = %v}", remoteState.Backend, remoteState.DisableInit, remoteState.DisableDependencyOptimization, remoteState.Generate, remoteState.Config) diff --git a/remote/remote_state_gcs.go b/remote/remote_state_gcs.go index 317c191fd..a3cec2c15 100644 --- a/remote/remote_state_gcs.go +++ b/remote/remote_state_gcs.go @@ -180,7 +180,7 @@ func (gcsInitializer GCSInitializer) Initialize(ctx context.Context, remoteState var gcsConfig = gcsConfigExtended.remoteStateConfigGCS cacheKey := gcsInitializer.buildInitializerCacheKey(&gcsConfig) - if initialized, hit := initializedRemoteStateCache.Get(cacheKey); initialized && hit { + if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit { terragruntOptions.Logger.Debugf("GCS bucket %s has already been confirmed to be initialized, skipping initialization checks", gcsConfig.Bucket) return nil } @@ -188,7 +188,7 @@ func (gcsInitializer GCSInitializer) Initialize(ctx context.Context, remoteState // ensure that only one goroutine can initialize bucket return stateAccessLock.StateBucketUpdate(gcsConfig.Bucket, func() error { // check if another goroutine has already initialized the bucket - if initialized, hit := initializedRemoteStateCache.Get(cacheKey); initialized && hit { + if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit { terragruntOptions.Logger.Debugf("GCS bucket %s has already been confirmed to be initialized, skipping initialization checks", gcsConfig.Bucket) return nil } @@ -211,7 +211,7 @@ func (gcsInitializer GCSInitializer) Initialize(ctx context.Context, remoteState } } - initializedRemoteStateCache.Put(cacheKey, true) + initializedRemoteStateCache.Put(ctx, cacheKey, true) return nil }) diff --git a/remote/remote_state_s3.go b/remote/remote_state_s3.go index a6a6932c6..9b8f89fe2 100644 --- a/remote/remote_state_s3.go +++ b/remote/remote_state_s3.go @@ -304,7 +304,7 @@ func (s3Initializer S3Initializer) Initialize(ctx context.Context, remoteState * var s3Config = s3ConfigExtended.remoteStateConfigS3 cacheKey := s3Initializer.buildInitializerCacheKey(&s3Config) - if initialized, hit := initializedRemoteStateCache.Get(cacheKey); initialized && hit { + if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit { terragruntOptions.Logger.Debugf("S3 bucket %s has already been confirmed to be initialized, skipping initialization checks", s3Config.Bucket) return nil } @@ -312,7 +312,7 @@ func (s3Initializer S3Initializer) Initialize(ctx context.Context, remoteState * // ensure that only one goroutine can initialize bucket return stateAccessLock.StateBucketUpdate(s3Config.Bucket, func() error { // Check if another goroutine has already initialized the bucket - if initialized, hit := initializedRemoteStateCache.Get(cacheKey); initialized && hit { + if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit { terragruntOptions.Logger.Debugf("S3 bucket %s has already been confirmed to be initialized, skipping initialization checks", s3Config.Bucket) return nil } @@ -352,7 +352,7 @@ func (s3Initializer S3Initializer) Initialize(ctx context.Context, remoteState * return errors.WithStackTrace(err) } - initializedRemoteStateCache.Put(cacheKey, true) + initializedRemoteStateCache.Put(ctx, cacheKey, true) return nil }) diff --git a/shell/context.go b/shell/context.go index 411712f33..f31322a01 100644 --- a/shell/context.go +++ b/shell/context.go @@ -3,18 +3,25 @@ package shell import ( "context" + "github.com/gruntwork-io/terragrunt/internal/cache" "github.com/gruntwork-io/terragrunt/util" "github.com/gruntwork-io/terragrunt/options" ) -const TerraformCommandContextKey ctxKey = iota +const ( + TerraformCommandContextKey ctxKey = iota + RunCmdCacheContextKey ctxKey = iota + + runCmdCacheName = "runCmdCache" +) type ctxKey byte type RunShellCommandFunc func(ctx context.Context, opts *options.TerragruntOptions, args []string) (*util.CmdOutput, error) func ContextWithTerraformCommandHook(ctx context.Context, fn RunShellCommandFunc) context.Context { + ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName)) return context.WithValue(ctx, TerraformCommandContextKey, fn) } diff --git a/shell/run_shell_cmd.go b/shell/run_shell_cmd.go index b033c0810..b98d15863 100644 --- a/shell/run_shell_cmd.go +++ b/shell/run_shell_cmd.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/gruntwork-io/terragrunt/engine" "github.com/gruntwork-io/terragrunt/telemetry" @@ -290,6 +292,11 @@ func (signalChannel *SignalsForwarder) Close() error { // GitTopLevelDir - fetch git repository path from passed directory func GitTopLevelDir(ctx context.Context, terragruntOptions *options.TerragruntOptions, path string) (string, error) { + runCache := cache.ContextCache[string](ctx, RunCmdCacheContextKey) + cacheKey := "top-level-dir-" + path + if gitTopLevelDir, found := runCache.Get(ctx, cacheKey); found { + return gitTopLevelDir, nil + } stdout := bytes.Buffer{} stderr := bytes.Buffer{} opts, err := options.NewTerragruntOptionsWithConfigPath(path) @@ -300,11 +307,13 @@ func GitTopLevelDir(ctx context.Context, terragruntOptions *options.TerragruntOp opts.Writer = &stdout opts.ErrWriter = &stderr cmd, err := RunShellCommandWithOutput(ctx, opts, path, true, false, "git", "rev-parse", "--show-toplevel") - terragruntOptions.Logger.Debugf("git show-toplevel result: \n%v\n%v\n", stdout.String(), stderr.String()) if err != nil { return "", err } - return strings.TrimSpace(cmd.Stdout), nil + cmdOutput := strings.TrimSpace(cmd.Stdout) + terragruntOptions.Logger.Debugf("git show-toplevel result: \n%v\n%v\n%v\n", stdout.String(), stderr.String(), cmdOutput) + runCache.Put(ctx, cacheKey, cmdOutput) + return cmdOutput, nil } // GitRepoTags - fetch git repository tags from passed url diff --git a/shell/run_shell_cmd_test.go b/shell/run_shell_cmd_test.go index d87f5aa80..29c37f701 100644 --- a/shell/run_shell_cmd_test.go +++ b/shell/run_shell_cmd_test.go @@ -6,6 +6,9 @@ import ( "strings" "testing" + "github.com/gruntwork-io/terragrunt/internal/cache" + "github.com/stretchr/testify/require" + "github.com/gruntwork-io/terragrunt/options" "github.com/stretchr/testify/assert" ) @@ -70,3 +73,21 @@ func TestLastReleaseTag(t *testing.T) { assert.NotEmpty(t, lastTag) assert.Equal(t, "v20.1.2", lastTag) } + +func TestGitLevelTopDirCaching(t *testing.T) { + t.Parallel() + ctx := context.Background() + ctx = ContextWithTerraformCommandHook(ctx, nil) + c := cache.ContextCache[string](ctx, RunCmdCacheContextKey) + assert.NotNil(t, c) + assert.Equal(t, 0, len(c.Cache)) + terragruntOptions, err := options.NewTerragruntOptionsForTest("") + require.NoError(t, err) + path := "." + path1, err := GitTopLevelDir(ctx, terragruntOptions, path) + require.NoError(t, err) + path2, err := GitTopLevelDir(ctx, terragruntOptions, path) + require.NoError(t, err) + assert.Equal(t, path1, path2) + assert.Equal(t, 1, len(c.Cache)) +} diff --git a/telemetry/metrics.go b/telemetry/metrics.go index 9ebd7bee0..49b3c7bf5 100644 --- a/telemetry/metrics.go +++ b/telemetry/metrics.go @@ -10,7 +10,6 @@ import ( "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" "github.com/gruntwork-io/go-commons/env" - "github.com/gruntwork-io/terragrunt/options" "github.com/pkg/errors" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" @@ -39,7 +38,7 @@ var metricNameCleanPattern = regexp.MustCompile(`[^A-Za-z0-9_.-/]`) var multipleUnderscoresPattern = regexp.MustCompile(`_+`) // Time - collect time for function execution -func Time(ctx context.Context, opts *options.TerragruntOptions, name string, attrs map[string]interface{}, fn func(childCtx context.Context) error) error { +func Time(ctx context.Context, name string, attrs map[string]interface{}, fn func(childCtx context.Context) error) error { if metricExporter == nil { return fn(ctx) } @@ -54,16 +53,16 @@ func Time(ctx context.Context, opts *options.TerragruntOptions, name string, att histogram.Record(ctx, time.Since(startTime).Milliseconds(), otelmetric.WithAttributes(metricAttrs...)) if err != nil { // count errors - Count(ctx, opts, ErrorsCounter, 1) - Count(ctx, opts, fmt.Sprintf("%s_errors", name), 1) + Count(ctx, ErrorsCounter, 1) + Count(ctx, fmt.Sprintf("%s_errors", name), 1) } else { - Count(ctx, opts, fmt.Sprintf("%s_success", name), 1) + Count(ctx, fmt.Sprintf("%s_success", name), 1) } return err } // Count - add to counter provided value -func Count(ctx context.Context, opts *options.TerragruntOptions, name string, value int64) { +func Count(ctx context.Context, name string, value int64) { if ctx == nil || metricExporter == nil { return } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 034298b07..44056bbf6 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -73,8 +73,8 @@ func ShutdownTelemetry(ctx context.Context) error { // Telemetry - collect telemetry from function execution - metrics and traces. func Telemetry(ctx context.Context, opts *options.TerragruntOptions, name string, attrs map[string]interface{}, fn func(childCtx context.Context) error) error { // wrap telemetry collection with trace and time metric - return Trace(ctx, opts, name, attrs, func(ctx context.Context) error { - return Time(ctx, opts, name, attrs, fn) + return Trace(ctx, name, attrs, func(ctx context.Context) error { + return Time(ctx, name, attrs, fn) }) } diff --git a/telemetry/traces.go b/telemetry/traces.go index 04b6d0c98..8031c766d 100644 --- a/telemetry/traces.go +++ b/telemetry/traces.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" - "github.com/gruntwork-io/terragrunt/options" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "github.com/gruntwork-io/go-commons/env" @@ -35,7 +33,7 @@ const ( ) // Trace - collect traces for method execution -func Trace(ctx context.Context, opts *options.TerragruntOptions, name string, attrs map[string]interface{}, fn func(childCtx context.Context) error) error { +func Trace(ctx context.Context, name string, attrs map[string]interface{}, fn func(childCtx context.Context) error) error { if spanExporter == nil || traceProvider == nil { // invoke function without tracing return fn(ctx) } diff --git a/test/fixture-get-repo-root/terragrunt.hcl b/test/fixture-get-repo-root/terragrunt.hcl index f961a4702..efd23202a 100644 --- a/test/fixture-get-repo-root/terragrunt.hcl +++ b/test/fixture-get-repo-root/terragrunt.hcl @@ -1,3 +1,4 @@ inputs = { repo_root = get_repo_root() + repo_root_2 = get_repo_root() } diff --git a/test/integration_test.go b/test/integration_test.go index 5e7ff6d42..0ba929176 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -7394,6 +7394,25 @@ func TestTerragruntLogSopsErrors(t *testing.T) { require.Contains(t, errorOut, "error base64-decoding encrypted data key: illegal base64 data at input byte") } +func TestGetRepoRootCaching(t *testing.T) { + t.Parallel() + cleanupTerraformFolder(t, TEST_FIXTURE_GET_REPO_ROOT) + tmpEnvPath, _ := filepath.EvalSymlinks(copyEnvironment(t, TEST_FIXTURE_GET_REPO_ROOT)) + rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_GET_REPO_ROOT) + + gitOutput, err := exec.Command("git", "init", rootPath).CombinedOutput() + if err != nil { + t.Fatalf("Error initializing git repo: %v\n%s", err, string(gitOutput)) + } + + stdout, stderr, err := runTerragruntCommandWithOutput(t, fmt.Sprintf("terragrunt run-all plan --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s", rootPath)) + require.NoError(t, err) + + output := fmt.Sprintf("%s %s", stdout, stderr) + count := strings.Count(output, "git show-toplevel result") + assert.Equal(t, 1, count) +} + func validateOutput(t *testing.T, outputs map[string]TerraformOutput, key string, value interface{}) { t.Helper() output, hasPlatform := outputs[key]