feat: enhance testing with sequential, parallel modes and flags for e…

…xceptions and skip-destroy (#40)

Co-authored-by: dkool <dkool@dkools-MacBook-Pro.local>

Makefile

@@ -1,23 +1,30 @@
-.PHONY: test docs fmt validate install-tools
-
-export EXAMPLE
+.PHONY: all install-tools validate fmt docs test test-parallel test-sequential
 
 all: install-tools validate fmt docs
 
 install-tools:
-	@go install github.com/terraform-docs/terraform-docs@latest
+	go install github.com/terraform-docs/terraform-docs@latest
+
+TEST_ARGS := $(if $(skip-destroy),-skip-destroy=$(skip-destroy)) \
+             $(if $(exception),-exception=$(exception)) \
+             $(if $(example),-example=$(example))
 
 test:
-	cd tests && go test -v -timeout 60m -run TestApplyNoError/$(EXAMPLE) ./deploy_test.go
+	cd tests && go test -v -timeout 60m -run '^TestApplyNoError$$' -args $(TEST_ARGS) .
+
+test-sequential:
+	cd tests && go test -v -timeout 120m -run '^TestApplyAllSequential$$' -args $(TEST_ARGS) .
+
+test-parallel:
+	cd tests && go test -v -timeout 60m -run '^TestApplyAllParallel$$' -args $(TEST_ARGS) .
 
 docs:
 	@echo "Generating documentation for root and modules..."
-	@BASE_DIR=$$(pwd); \
-	terraform-docs markdown . --output-file $$BASE_DIR/README.md --output-mode inject --hide modules; \
-	for dir in $$BASE_DIR/modules/*; do \
+	terraform-docs markdown . --output-file README.md --output-mode inject --hide modules
+	for dir in modules/*; do \
 		if [ -d "$$dir" ]; then \
 			echo "Processing $$dir..."; \
-			terraform-docs markdown $$dir --output-file $$dir/README.md --output-mode inject --hide modules || echo "Skipped: $$dir"; \
+			terraform-docs markdown "$$dir" --output-file "$$dir/README.md" --output-mode inject --hide modules || echo "Skipped: $$dir"; \
 		fi \
 	done
 
@@ -28,7 +35,4 @@ validate:
 	terraform init -backend=false
 	terraform validate
 	@echo "Cleaning up initialization files..."
-	@rm -rf .terraform
-	@rm -f terraform.tfstate
-	@rm -f terraform.tfstate.backup
-	@rm -f .terraform.lock.hcl
+	rm -rf .terraform terraform.tfstate terraform.tfstate.backup .terraform.lock.hcl

README.md

@@ -87,13 +87,7 @@ End-to-end testing is not conducted on these modules, as they are individual com
 
 ## Testing
 
-Ensure go and terraform are installed.
-
-Run tests for different usage scenarios by specifying the EXAMPLE environment variable. Usage examples are in the examples directory.
-
-To execute a test, run `make test EXAMPLE=default`
-
-Replace default with the specific example you want to test. These tests ensure the module performs reliably across various configurations.
+For more information, please see our testing [guidelines](./TESTING.md)
 
 ## Notes
 

TESTING.md

@@ -0,0 +1,13 @@
+## Testing
+
+Ensure Go and Terraform are installed.
+
+Run tests for different scenarios by setting the example flag when running the tests.
+
+To run a test, use make test example=default, replacing default with the example you want to test from the examples directory.
+
+Add skip-destroy=true to skip the destroy step, like make test example=default skip-destroy=true.
+
+For running all tests in parallel or sequentially with make test-parallel or make test-sequential, exclude specific examples by adding exception=example1,example2, where example1 and example2 are examples to skip.
+
+These tests ensure the module's reliability across configurations.

tests/deploy_test.go

@@ -1,21 +1,57 @@
 package main
 
 import (
+	"flag"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/gruntwork-io/terratest/modules/terraform"
 )
 
-type TerraformModule struct {
+var (
+	skipDestroy   bool
+	exception     string
+	example       string
+	exceptionList map[string]bool
+)
+
+func init() {
+	flag.BoolVar(&skipDestroy, "skip-destroy", false, "Skip running terraform destroy after apply")
+	flag.StringVar(&exception, "exception", "", "Comma-separated list of examples to exclude")
+	flag.StringVar(&example, "example", "", "Specific example to test")
+}
+
+func parseExceptionList() {
+	exceptionList = make(map[string]bool)
+	if exception != "" {
+		examples := strings.Split(exception, ",")
+		for _, ex := range examples {
+			exceptionList[strings.TrimSpace(ex)] = true
+		}
+	}
+}
+
+type Module struct {
 	Name    string
 	Path    string
 	Options *terraform.Options
 }
 
-func NewTerraformModule(name, path string) *TerraformModule {
-	return &TerraformModule{
+type ModuleManager struct {
+	BaseExamplesPath string
+}
+
+func NewModuleManager(baseExamplesPath string) *ModuleManager {
+	return &ModuleManager{
+		BaseExamplesPath: baseExamplesPath,
+	}
+}
+
+func NewModule(name, path string) *Module {
+	return &Module{
 		Name: name,
 		Path: path,
 		Options: &terraform.Options{
@@ -25,48 +61,169 @@ func NewTerraformModule(name, path string) *TerraformModule {
 	}
 }
 
-func (m *TerraformModule) Apply(t *testing.T) {
+func (mm *ModuleManager) DiscoverModules() ([]*Module, error) {
+	var modules []*Module
+
+	entries, err := os.ReadDir(mm.BaseExamplesPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read examples directory: %v", err)
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			moduleName := entry.Name()
+			if exceptionList[moduleName] {
+				fmt.Printf("Skipping module %s as it is in the exception list\n", moduleName)
+				continue
+			}
+			modulePath := filepath.Join(mm.BaseExamplesPath, moduleName)
+			modules = append(modules, NewModule(moduleName, modulePath))
+		}
+	}
+
+	return modules, nil
+}
+
+func (m *Module) Apply(t *testing.T) error {
+	t.Helper()
 	t.Logf("Applying Terraform module: %s", m.Name)
 	terraform.WithDefaultRetryableErrors(t, m.Options)
-	terraform.InitAndApply(t, m.Options)
+	_, err := terraform.InitAndApplyE(t, m.Options)
+	return err
 }
 
-func (m *TerraformModule) Destroy(t *testing.T) {
+func (m *Module) Destroy(t *testing.T) error {
+	t.Helper()
 	t.Logf("Destroying Terraform module: %s", m.Name)
-	terraform.Destroy(t, m.Options)
-	m.cleanupFiles(t)
+	_, err := terraform.DestroyE(t, m.Options)
+	if err != nil {
+		return fmt.Errorf("destroy failed: %v", err)
+	}
+	if err := m.cleanupFiles(t); err != nil {
+		return fmt.Errorf("cleanup failed: %v", err)
+	}
+	return nil
 }
 
-func (m *TerraformModule) cleanupFiles(t *testing.T) {
+func (m *Module) cleanupFiles(t *testing.T) error {
+	t.Helper()
 	t.Logf("Cleaning up in: %s", m.Options.TerraformDir)
-	filesToCleanup := []string{"*.terraform*", "*tfstate*"}
+	filesToCleanup := []string{"*.terraform*", "*tfstate*", "*.lock.hcl"}
+
 	for _, pattern := range filesToCleanup {
 		matches, err := filepath.Glob(filepath.Join(m.Options.TerraformDir, pattern))
 		if err != nil {
-			t.Errorf("Error matching pattern %s: %v", pattern, err)
-			continue
+			return fmt.Errorf("error matching pattern %s: %v", pattern, err)
 		}
 		for _, filePath := range matches {
 			if err := os.RemoveAll(filePath); err != nil {
-				t.Errorf("Failed to remove %s: %v", filePath, err)
+				return fmt.Errorf("failed to remove %s: %v", filePath, err)
 			}
 		}
 	}
+	return nil
+}
+
+func RunTests(t *testing.T, modules []*Module, parallel bool) {
+	// Error collector to accumulate failures and reasons
+	var errorMessages []string
+
+	for _, module := range modules {
+		module := module
+		t.Run(module.Name, func(t *testing.T) {
+			if parallel {
+				t.Parallel()
+			}
+
+			// Defer Destroy to ensure cleanup happens, regardless of Apply success or failure
+			if !skipDestroy {
+				defer func() {
+					if err := module.Destroy(t); err != nil {
+						t.Logf("Warning: Cleanup for module %s failed: %v", module.Name, err)
+					}
+				}()
+			}
+
+			// Apply the module and collect errors
+			if err := module.Apply(t); err != nil {
+				// Mark this test as failed and collect the error message
+				t.Fail()
+				errorMessages = append(errorMessages, fmt.Sprintf("Module %s failed: %v", module.Name, err))
+				t.Logf("Apply failed for module %s: %v", module.Name, err)
+			}
+		})
+	}
+
+	// After all tests are complete, log the summary of errors if any
+	t.Cleanup(func() {
+		if len(errorMessages) > 0 {
+			t.Log("Summary of failed modules:")
+			for _, msg := range errorMessages {
+				t.Log(msg)
+			}
+		} else {
+			t.Log("All modules applied and destroyed successfully.")
+		}
+	})
 }
 
 func TestApplyNoError(t *testing.T) {
-	t.Parallel()
+	flag.Parse()
+	parseExceptionList()
 
-	example := os.Getenv("EXAMPLE")
 	if example == "" {
-		t.Fatal("EXAMPLE environment variable is not set")
+		t.Fatal("-example flag is not set")
+	}
+
+	if exceptionList[example] {
+		t.Skipf("Skipping example %s as it is in the exception list", example)
 	}
 
 	modulePath := filepath.Join("..", "examples", example)
-	module := NewTerraformModule(example, modulePath)
+	module := NewModule(example, modulePath)
+	var errorMessages []string
 
-	t.Run(module.Name, func(t *testing.T) {
-		defer module.Destroy(t)
-		module.Apply(t)
-	})
+	if err := module.Apply(t); err != nil {
+		errorMessages = append(errorMessages, fmt.Sprintf("Apply failed for module %s: %v", module.Name, err))
+		t.Fail()
+	}
+
+	if !skipDestroy {
+		if err := module.Destroy(t); err != nil {
+			errorMessages = append(errorMessages, fmt.Sprintf("Cleanup failed for module %s: %v", module.Name, err))
+		}
+	}
+
+	if len(errorMessages) > 0 {
+		fmt.Println("Summary of errors:")
+		for _, msg := range errorMessages {
+			fmt.Println(msg)
+		}
+	}
+}
+
+func TestApplyAllSequential(t *testing.T) {
+	flag.Parse()
+	parseExceptionList()
+
+	manager := NewModuleManager(filepath.Join("..", "examples"))
+	modules, err := manager.DiscoverModules()
+	if err != nil {
+		t.Fatalf("Failed to discover modules: %v", err)
+	}
+
+	RunTests(t, modules, false)
+}
+
+func TestApplyAllParallel(t *testing.T) {
+	flag.Parse()
+	parseExceptionList()
+
+	manager := NewModuleManager(filepath.Join("..", "examples"))
+	modules, err := manager.DiscoverModules()
+	if err != nil {
+		t.Fatalf("Failed to discover modules: %v", err)
+	}
+
+	RunTests(t, modules, true)
 }