Merge pull request #82 from gruntwork-io/fix-spin-up-subfolders

Fix spin-up missing deeper subfolders. Fix AWS API getting overloaded during tests.

glide.yaml

@@ -8,6 +8,7 @@ import:
 - package: github.com/stretchr/testify/assert
 - package: github.com/go-errors/errors
 - package: github.com/mitchellh/mapstructure
+- package: github.com/mattn/go-zglob
 - package: github.com/aws/aws-sdk-go
   subpackages:
   - aws

locks/dynamodb/dynamo_lock_table.go

@@ -8,6 +8,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/gruntwork-io/terragrunt/errors"
 	"github.com/gruntwork-io/terragrunt/options"
+	"github.com/gruntwork-io/terragrunt/util"
 )
 
 // Create the lock table in DynamoDB if it doesn't already exist
@@ -82,6 +83,13 @@ func isTableAlreadyBeingCreatedError(err error) bool {
 // Wait for the given DynamoDB table to be in the "active" state. If it's not in "active" state, sleep for the
 // specified amount of time, and try again, up to a maximum of maxRetries retries.
 func waitForTableToBeActive(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetries time.Duration, terragruntOptions *options.TerragruntOptions) error {
+	return waitForTableToBeActiveWithRandomSleep(tableName, client, maxRetries, sleepBetweenRetries, sleepBetweenRetries, terragruntOptions)
+}
+
+// Waits for the given table as described above, but sleeps a random amount of time greater than sleepBetweenRetriesMin
+// and less than sleepBetweenRetriesMax between tries. This is to avoid an AWS issue where all waiting requests fire at
+// the same time, which continually triggered AWS's "subscriber limit exceeded" API error.
+func waitForTableToBeActiveWithRandomSleep(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetriesMin time.Duration, sleepBetweenRetriesMax time.Duration, terragruntOptions *options.TerragruntOptions) error {
 	for i := 0; i < maxRetries; i++ {
 		tableReady, err := lockTableExistsAndIsActive(tableName, client)
 		if err != nil {
@@ -93,6 +101,7 @@ func waitForTableToBeActive(tableName string, client *dynamodb.DynamoDB, maxRetr
 			return nil
 		}
 
+		sleepBetweenRetries := util.GetRandomTime(sleepBetweenRetriesMin, sleepBetweenRetriesMax)
 		terragruntOptions.Logger.Printf("Table %s is not yet in active state. Will check again after %s.", tableName, sleepBetweenRetries)
 		time.Sleep(sleepBetweenRetries)
 	}

locks/dynamodb/dynamo_lock_table_test.go

@@ -52,7 +52,7 @@ func TestWaitForTableToBeActiveTableDoesNotExist(t *testing.T) {
 	tableName := "table-does-not-exist"
 	retries := 5
 
-	err := waitForTableToBeActive(tableName, client, retries, 1 * time.Millisecond, mockOptions)
+	err := waitForTableToBeActiveWithRandomSleep(tableName, client, retries, 1 * time.Millisecond, 500 * time.Millisecond, mockOptions)
 
 	assert.True(t, errors.IsError(err, TableActiveRetriesExceeded{TableName: tableName, Retries: retries}), "Unexpected error of type %s: %s", reflect.TypeOf(err), err)
 }

spin/stack.go

@@ -1,12 +1,14 @@
 package spin
 
 import (
-	"github.com/gruntwork-io/terragrunt/options"
 	"fmt"
-	"path/filepath"
+	"strings"
+
 	"github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/errors"
-	"strings"
+	"github.com/gruntwork-io/terragrunt/options"
+
+	"github.com/mattn/go-zglob"
 )
 
 // Represents a stack of Terraform modules (i.e. folders with Terraform templates) that you can "spin up" or
@@ -47,7 +49,10 @@ func (stack *Stack) CheckForCycles() error {
 // Find all the Terraform modules in the subfolders of the working directory of the given TerragruntOptions and
 // assemble them into a Stack object that can be applied or destroyed in a single command
 func FindStackInSubfolders(terragruntOptions *options.TerragruntOptions) (*Stack, error) {
-	terragruntConfigFiles, err := filepath.Glob(fmt.Sprintf("%s/**/%s", terragruntOptions.WorkingDir, config.DefaultTerragruntConfigPath))
+	// Ideally, we'd use a builin Go library like filepath.Glob here, but per https://github.com/golang/go/issues/11862,
+	// the current go implementation doesn't support treating ** as zero or more directories, just zero or one.
+	// So we use a third-party library.
+	terragruntConfigFiles, err := zglob.Glob(fmt.Sprintf("%s/**/%s", terragruntOptions.WorkingDir, config.DefaultTerragruntConfigPath))
 	if err != nil {
 		return nil, errors.WithStackTrace(err)
 	}

spin/stack_test.go

@@ -0,0 +1,84 @@
+package spin
+
+import (
+	"testing"
+	"io/ioutil"
+	"path/filepath"
+	"os"
+	"github.com/gruntwork-io/terragrunt/options"
+	"github.com/stretchr/testify/assert"
+	"github.com/gruntwork-io/terragrunt/util"
+	"strings"
+)
+
+func TestFindStackInSubfolders(t *testing.T) {
+	t.Parallel()
+
+	filePaths := []string{
+		"/stage/data-stores/redis/.terragrunt",
+		"/stage/data-stores/postgres/.terragrunt",
+		"/stage/ecs-cluster/.terragrunt",
+		"/stage/kms-master-key/.terragrunt",
+		"/stage/vpc/.terragrunt",
+	}
+
+	tempFolder := createTempFolder(t)
+	writeAsEmptyFiles(t, tempFolder, filePaths)
+
+	envFolder := filepath.Join(tempFolder + "/stage")
+	terragruntOptions := options.NewTerragruntOptions(envFolder)
+	terragruntOptions.WorkingDir = envFolder
+
+	stack, err := FindStackInSubfolders(terragruntOptions)
+	if err != nil {
+		t.Fatalf("Failed when calling method under test: %s\n", err.Error())
+	}
+
+	var modulePaths []string
+
+	for _, module := range stack.Modules {
+		relPath := strings.Replace(module.Path, tempFolder, "", 1)
+		relPath = filepath.Join(relPath, ".terragrunt")
+
+		modulePaths = append(modulePaths, relPath)
+	}
+
+	for _, filePath := range filePaths {
+		filePathFound := util.ListContainsElement(modulePaths, filePath)
+		assert.True(t, filePathFound, "The filePath %s was not found by Terragrunt.\n", filePath)
+	}
+
+}
+
+func createTempFolder(t *testing.T) string {
+	tmpFolder, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("Failed to create temp directory: %s\n", err.Error())
+	}
+
+	return tmpFolder
+}
+
+// Create an empty file at each of the given paths
+func writeAsEmptyFiles(t *testing.T, tmpFolder string, paths []string) {
+	for _, path := range paths {
+		absPath := filepath.Join(tmpFolder, path)
+
+		containingDir := filepath.Dir(absPath)
+		createDirIfNotExist(t, containingDir)
+
+		err := ioutil.WriteFile(absPath, nil, os.ModePerm)
+		if err != nil {
+			t.Fatalf("Failed to write file at path %s: %s\n", path, err.Error())
+		}
+	}
+}
+
+func createDirIfNotExist(t *testing.T, path string) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		err = os.MkdirAll(path, os.ModePerm)
+		if err != nil {
+			t.Fatalf("Failed to create directory: %s\n", err.Error())
+		}
+	}
+}

util/random.go

@@ -0,0 +1,42 @@
+package util
+
+import (
+	"time"
+	"math/rand"
+)
+
+// Get a random time duration between the lower bound and upper bound. This is useful because some of our automated tests
+// wound up flooding the AWS API all at once, leading to a "Subscriber limit exceeded" error.
+// TODO: Some of the more exotic test cases fail, but it's not worth catching them given the intended use of this function.
+func GetRandomTime(lowerBound, upperBound time.Duration) time.Duration {
+	if lowerBound < 0 {
+		lowerBound = -1 * lowerBound
+	}
+
+	if upperBound < 0 {
+		upperBound = -1 * upperBound
+	}
+
+	if lowerBound > upperBound {
+		return upperBound
+	}
+
+	if lowerBound == upperBound {
+		return lowerBound
+	}
+
+	lowerBoundMs := lowerBound.Seconds() * 1000
+	upperBoundMs := upperBound.Seconds() * 1000
+
+	lowerBoundMsInt := int(lowerBoundMs)
+	upperBoundMsInt := int(upperBoundMs)
+
+	randTimeInt := random(lowerBoundMsInt, upperBoundMsInt)
+	return time.Duration(randTimeInt) * time.Millisecond
+}
+
+// Generate a random int between min and max, inclusive
+func random(min int, max int) int {
+	rand.Seed(time.Now().UnixNano())
+	return rand.Intn(max - min) + min
+}

util/random_test.go

@@ -0,0 +1,40 @@
+package util
+
+import (
+	"testing"
+	"time"
+)
+
+func TestGetRandomTime(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct{
+		lowerBound time.Duration
+		upperBound time.Duration
+	}{
+		{ 1 * time.Second, 10 * time.Second },
+		{0, 0},
+		{-1 * time.Second, -3 * time.Second},
+		{1 * time.Second, 2000000001 * time.Nanosecond},
+		{1 * time.Millisecond, 10 * time.Millisecond},
+		//{1 * time.Second, 1000000001 * time.Nanosecond}, // This case fails
+	}
+
+	// Loop through each test case
+	for _, testCase := range testCases {
+		// Try each test case 100 times to avoid fluke test results
+		for i := 0; i < 100; i++ {
+			actual := GetRandomTime(testCase.lowerBound, testCase.upperBound)
+
+			if  testCase.lowerBound > 0 && testCase.upperBound > 0 {
+				if actual < testCase.lowerBound {
+					t.Fatalf("Randomly computed time %v should not be less than lowerBound %v", actual, testCase.lowerBound)
+				}
+
+				if actual > testCase.upperBound {
+					t.Fatalf("Randomly computed time %v should not be greater than upperBound %v", actual, testCase.upperBound)
+				}
+			}
+		}
+	}
+}