From 598f0c7be4fd69de0b598bd83db28ca8960cf715 Mon Sep 17 00:00:00 2001 From: Matt Gowie Date: Thu, 26 Dec 2024 14:17:42 -0700 Subject: [PATCH] feat: begins the work to support single vs multi root_module_structure (#17) ## what - This is a first cut at supporting `MultiInstance` vs `SingleInstance` root module structures. - Adds an additional example for the single-instance work. - Both examples are now applied to our masterpoint Spacelift tenant. - Work on the README to explain Multi vs Single. Please read and provide critical feedback. - Includes an update to renovate to automerge PRs for non major revs. ## why - People structure their terraform and root modules in different ways. We need to support both. ## references - Some discussion in Slack: https://masterpoint.slack.com/archives/C04MUCKUDKK/p1734543200130119 - Original design session with @gberenice + @oycyc on this topic: https://app.fireflies.ai/view/Spacelift-Automation-Airtable-Design::cNXp0mkZEAUuzhLm ## Summary by CodeRabbit - **New Features** - Introduced automated merging of specific dependency updates in configuration. - Added support for AWS integration with new settings. - New variable for configuring the length of generated names. - New module for automation with AWS integration settings. - Enhanced stack settings for better management and description. - **Bug Fixes** - Updated variable names for clarity and consistency. - **Documentation** - Improved README documentation for better clarity on module usage and structure. - Added examples and distinctions between MultiInstance and SingleInstance structures. - **Chores** - Updated GitHub Actions workflow with a new environment variable for API key endpoint. - Removed outdated backend configuration file. --------- Co-authored-by: Veronika Gnilitska <30597968+gberenice@users.noreply.github.com> --- .github/renovate.json5 | 5 + .github/workflows/tf-test.yaml | 1 + README.md | 135 ++++++++++++++---- .../stacks/{example.yaml => dev.yaml} | 0 .../tfvars/{example.tfvars => dev.tfvars} | 0 .../spacelift-automation/backend.tf.json | 18 --- .../components/spacelift-automation/main.tf | 3 +- .../spacelift-automation/stacks/common.yaml | 1 - .../tfvars/example.tfvars | 2 +- .../random-pet/default.auto.tfvars | 1 + .../root-modules/random-pet/main.tf | 3 + .../root-modules/random-pet/stack.yaml | 7 + .../root-modules/random-pet/variables.tf | 4 + .../root-modules/random-pet/versions.tf | 10 ++ .../root-modules/rds-cluster-dev/main.tf | 5 + .../root-modules/rds-cluster-dev/stack.yaml | 2 + .../root-modules/rds-cluster-dev/versions.tf | 10 ++ .../root-modules/rds-cluster-prod/main.tf | 5 + .../root-modules/rds-cluster-prod/stack.yaml | 2 + .../root-modules/rds-cluster-prod/versions.tf | 10 ++ .../spacelift-automation-example2/main.tf | 15 ++ .../spacelift-automation-example2/stack.yaml | 8 ++ .../spacelift-automation-example2/versions.tf | 10 ++ main.tf | 106 ++++++++------ .../root-module-a/stacks/common.yaml | 0 .../root-module-a/stacks/default-example.yaml | 0 .../root-module-a/stacks/test.yaml | 0 .../tfvars/default-example.tfvars | 0 .../root-module-a/tfvars/test.tfvars | 0 .../single-instance/root-module-a/stack.yaml | 6 + tests/main.tftest.hcl | 6 +- tests/single-instance.tftest.hcl | 130 +++++++++++++++++ variables.tf | 19 ++- 33 files changed, 423 insertions(+), 101 deletions(-) rename examples/complete/components/random-pet/stacks/{example.yaml => dev.yaml} (100%) rename examples/complete/components/random-pet/tfvars/{example.tfvars => dev.tfvars} (100%) delete mode 100644 examples/complete/components/spacelift-automation/backend.tf.json create mode 100644 examples/single-instance/root-modules/random-pet/default.auto.tfvars create mode 100644 examples/single-instance/root-modules/random-pet/main.tf create mode 100644 examples/single-instance/root-modules/random-pet/stack.yaml create mode 100644 examples/single-instance/root-modules/random-pet/variables.tf create mode 100644 examples/single-instance/root-modules/random-pet/versions.tf create mode 100644 examples/single-instance/root-modules/rds-cluster-dev/main.tf create mode 100644 examples/single-instance/root-modules/rds-cluster-dev/stack.yaml create mode 100644 examples/single-instance/root-modules/rds-cluster-dev/versions.tf create mode 100644 examples/single-instance/root-modules/rds-cluster-prod/main.tf create mode 100644 examples/single-instance/root-modules/rds-cluster-prod/stack.yaml create mode 100644 examples/single-instance/root-modules/rds-cluster-prod/versions.tf create mode 100644 examples/single-instance/root-modules/spacelift-automation-example2/main.tf create mode 100644 examples/single-instance/root-modules/spacelift-automation-example2/stack.yaml create mode 100644 examples/single-instance/root-modules/spacelift-automation-example2/versions.tf rename tests/fixtures/{ => multi-instance}/root-module-a/stacks/common.yaml (100%) rename tests/fixtures/{ => multi-instance}/root-module-a/stacks/default-example.yaml (100%) rename tests/fixtures/{ => multi-instance}/root-module-a/stacks/test.yaml (100%) rename tests/fixtures/{ => multi-instance}/root-module-a/tfvars/default-example.tfvars (100%) rename tests/fixtures/{ => multi-instance}/root-module-a/tfvars/test.tfvars (100%) create mode 100644 tests/fixtures/single-instance/root-module-a/stack.yaml create mode 100644 tests/single-instance.tftest.hcl diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a485de5..a53184a 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -8,6 +8,11 @@ "baseBranches": ["main", "master"], "labels": ["auto-upgrade"], "dependencyDashboardAutoclose": true, + "packageRules": [{ + // Allow auto merge if it's not a major version update + "matchUpdateTypes": ["minor", "patch", "pin", "digest"], + "automerge": true + }], "terraform": { "ignorePaths": [ "**/context.tf", diff --git a/.github/workflows/tf-test.yaml b/.github/workflows/tf-test.yaml index 8fda6d4..f60c8d6 100644 --- a/.github/workflows/tf-test.yaml +++ b/.github/workflows/tf-test.yaml @@ -7,6 +7,7 @@ on: pull_request: env: + SPACELIFT_API_KEY_ENDPOINT: ${{ secrets.SPACELIFT_API_KEY_ENDPOINT }} SPACELIFT_API_KEY_ID: ${{ secrets.SPACELIFT_API_KEY_ID }} SPACELIFT_API_KEY_SECRET: ${{ secrets.SPACELIFT_API_KEY_SECRET }} diff --git a/README.md b/README.md index 91b3c1b..c92c61e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This Terraform child module provides infrastructure automation for projects in [ ## Overview -The `spacelift-automation` root module is designed to streamline the deployment and management of all Spacelift infrastructure, including itself. +The `spacelift-automation` root module is designed to streamline the deployment and management of all Spacelift infrastructure, including creating a Spacelift Stack to manage itself. It automates the creation of "child" stacks and all the required accompanying Spacelift resources. For each enabled root module it creates: @@ -17,22 +17,27 @@ It automates the creation of "child" stacks and all the required accompanying Sp 3. [Spacelift AWS Integration Attachment](https://docs.spacelift.io/integrations/cloud-providers/aws#lets-explain) Associates a specific AWS IAM role with a stack to allow it to assume that role. The IAM role typically has permissions to manage specific AWS resources, and Spacelift assumes this role to run the operations required by the stack. 4. [Spacelift Initialization Hook](https://docs.spacelift.io/concepts/run#initializing) - Prepares your environment before executing infrastructure code. This custom script copies corresponding Terraform tfvars files into a working directory before either run or task as a `spacelift.auto.tfvars` file. It's [automatically loaded](https://opentofu.org/docs/v1.7/language/values/variables/#variable-definitions-tfvars-files) into the OpenTofu/Terraform execution environment. + Prepares your environment before executing infrastructure code. This custom script copies corresponding Terraform tfvars files into a working directory before any Spacelift run or task as a `spacelift.auto.tfvars` file. This ensures your tfvars are [automatically loaded](https://opentofu.org/docs/v1.7/language/values/variables/#variable-definitions-tfvars-files) into the OpenTofu/Terraform execution environment. ## Usage -Spacelift Automation logic is opinionated and heavily relies on the Git repository structure. -This module is configured to track all the files in the provided root module directory and create the stack based on the provided configuration (if any). +Spacelift Automation logic is opinionated and heavily relies on certain repository structures. +This module is configured to track all the files in the given root module directory and create Spacelift Stacks based on the provided configuration. -Structure requirements are: +We support the following root module directory structures, which are controlled by the `var.root_modules_structure` variable: -- Stack configs are placed in `/stacks` directory. -- Terraform variables are placed in `/tfvars` directory. -- Stack config file and tfvars file must be equal to OpenTofu/Terraform workspace, e.g. `dev.yaml` and `dev.tfvars`. -- Common configs are placed in `/stacks/common.yaml` file. This is useful when you know that some values should be shared across all the stacks created for a root module, e.g. all stacks that manage Spacelift Policy must be Administrative. You can override this file name using Terraform variable. +### `MultiInstance` (the default) -Let's check the example. -Input repo structure: +This is the default structure that we expect and recommend. This is intended for root modules that manage multiple state files (instances) through [workspaces](https://opentofu.org/docs/cli/workspaces/) or [Dynamic Backend configurations](https://opentofu.org/docs/intro/whats-new/#early-variablelocals-evaluation). + +Structure requirements: + +- Stack configs are placed in `//stacks` directory for each workspace / instance of that stack. e.g. `root-modules/k8s-cluster/stacks/dev.yaml` and `root-modules/k8s-cluster/stacks/stage.yaml` +- Terraform variables are placed in `//tfvars` directory for each workspace / instance of that stack. e.g. `root-modules/k8s-cluster/tfvars/dev.tfvars` and `root-modules/k8s-cluster/tfvars/stage.tfvars` +- Stack config files and tfvars files must be equal to OpenTofu/Terraform workspace, e.g. `stacks/dev.yaml` and `tfvars/dev.tfvars` for a workspace named `dev`. +- Common configs are placed in `//stacks/common.yaml` file (or `var.common_config_file` value). This is useful when you know that some values should be shared across all the stacks created for a root module. For example, all stacks that manage Spacelift Policies must use the `administrative: true` setting or all stacks must share the same labels. + +We have an example of this structure in the [examples/complete](./examples/complete/components/), which looks like the following: ```sh ├── root-modules @@ -45,6 +50,7 @@ Input repo structure: │ │ │ └── dev.tfvars │ │ │ └── stage.tfvars │ │ ├── variables.tf +│ │ ├── main.tf │ │ └── versions.tf │ ├── k8s-cluster │ │ ├── stacks @@ -60,26 +66,27 @@ Input repo structure: ... ``` -Root module inputs: +The `spacelift-automation/main.tf` file looks something like this: ```hcl -aws_integration_id = "ZDPP8SKNVG0G27T4" - -# GitHub configuration github_enterprise = { namespace = "masterpointio" } repository = "terraform-spacelift-automation" # Stacks configurations -root_modules_path = "root-modules" -enabled_root_modules = ["spacelift-aws-role"] +root_modules_path = "root-modules" +all_root_modules_enabled = true + +aws_integration_id = "ZDPP8SKNVG0G27T4" ``` The configuration above creates the following stacks: - `spacelift-aws-role-dev` - `spacelift-aws-role-stage` +- `k8s-cluster-dev` +- `k8s-cluster-prod` These stacks have the following configuration: @@ -88,29 +95,101 @@ These stacks have the following configuration: - Corresponding Terraform variables are generated by an [Initialization Hook](https://docs.spacelift.io/concepts/run#initializing) and placed in the root of each Stack's working directory during each run or task. For example, the content of the file `root-modules/spacelift-aws-role/tfvars/dev.tfvars` will be copied to working directory of the Stack `spacelift-aws-role-dev` as file `spacelift.auto.tfvars` allowing the OpenTofu/Terraform inputs to be automatically loaded. - If you would like to disable this functionality, you can set `tfvars.enabled` in the Stack's YAML file to `false`. -## FAQs +### `SingleInstance` -### Why are variable values provided separately in `tfvars/` and not in the `yaml` file? +This is a special case where each root module directory only manages one state file (instance). Each time you want to create a new instance of a root module, you need to create a new directory with the same code and change your inputs. **We do not recommend this structure** as it is less flexible and easily leads to anti-patterns, but it is supported. + +Structure requirements: + +- Stack configs are placed in `//stack.yaml` directory. e.g. `root-modules/rds-cluster/stack.yaml` +- Tfvars values are not supported in this structure. In this structure, we suggest you just add your tfvars as `***.auto.tfvars` or hardcode your values directly in root module code. -This is to support easy local and outside-spacelift operations. Keeping variable values in a `tfvars` file per workspace allows you to simply pass that file to the relevant CLI command locally via the `-var-file` option so that you don't need to provide values individually. +Here is an example of this structure that we have in the [examples/single-instance](./examples/single-instance/) directory: + +```sh +├── root-modules +│ ├── spacelift-automation +│ │ ├── stack.yaml +│ │ ├── variables.tf +│ │ ├── main.tf +│ │ └── versions.tf +│ ├── rds-cluster-dev +│ │ ├── stack.yaml +│ │ ├── main.tf +│ │ └── versions.tf +│ ├── rds-cluster-prod +│ │ ├── stack.yaml +│ │ ├── main.tf +│ │ └── versions.tf +│ ├── random-pet +│ │ ├── stack.yaml +│ │ ├── variables.tf +│ │ ├── main.tf +│ │ └── versions.tf +... +``` + +The configuration above creates the following Spacelift Stacks: + +- `spacelift-automation` +- `rds-cluster-dev` +- `rds-cluster-prod` +- `random-pet` + +These stacks will be configured using the settings in the `stack.yaml` file. + +## FAQs ### Can I create a Spacelift Stack for Spacelift Automation? (Recommended) -Spacelift Automation can manage itself as a Stack as well, and we recommend this so you can fully automate your Stack management upon merging to your given branch. Follow these next steps to achieve that: +Spacelift Automation can manage itself as a Stack as well, and we recommend this so you can fully automate your Stack management upon merging to your given branch. Follow these steps to achieve that: + +1. Create a new vanilla OpenTofu/Terraform root module in `/spacelift-automation` that consumes this child module and supplies the necessary configuration for your unique setup. e.g. + + ```hcl + # root-modules/spacelift-automation/main.tf + + module "spacelift-automation" { + source = "masterpointio/automation/spacelift" + version = "x.x.x" # Always pin a version, use the latest version from the release page. + + # GitHub configuration + github_enterprise = { + namespace = "masterpointio" + } + repository = "your-infrastructure-repo" + + # Stacks configurations + root_modules_path = "../../root-modules" + all_root_modules_enabled = true + + aws_integration_id = "ZDPP8SKNVG0G27T4" + } + ``` -1. Create a new vanilla OpenTofu/Terraform root module that consumes this child module and supplies the necessary configuration for your unique setup. In other words, it's a configuration that uses the default capabilities of either OpenTofu or Terraform without any customization, or third-party tools or plugins. 2. Optionally, create a Terraform workspace that will be used for your Automation configuration, e.g.: + ```sh - tofu workspace new masterpoint + tofu workspace new main ``` - Remember that Stack config and tfvars file name must be equal to the workspace, which can be `default`. -3. Apply the vanilla OpenTofu/Terraform configuration. + + Remember that Stack config and tfvars file name must be equal to the workspace e.g. `main.yaml` and `main.tfvars`. If you choose not to create a new workspace, this can be `default.yaml` and `default.tfvars`. + +3. Apply the `spacelift-automation` root module. 4. Move the Automation configs to the `/spacelift-automation/stacks` directory and push the changes to the tracked repo and branch. -5. From this moment, Spacelift Automation is tracking the changes to its Stack configs and Terraform variables. +5. After pushed to your repo's tracked branch, Spacelift Automation will track the addition of new root modules and create Stacks for them. -Check out an example of such a configuration in the [examples/complete](./examples/complete/components/spacelift-automation/tfvars/example.tfvars). +Check out an example configuration in the [examples/complete](./examples/complete/components/spacelift-automation/tfvars/example.tfvars). + + + +### What goes in a Stack config file? e.g. `stacks/dev.yaml`, `stacks/common.yaml`, `stack.yaml`, etc + +Most settings that you would set on [the Spacelift Stack resource](https://search.opentofu.org/provider/spacelift-io/spacelift/latest/docs/resources/stack) are supported. Additionally, you can include certain Stack specific settings that will override this module's defaults like `default_tf_workspace_enabled`, `tfvars.enabled`, and similar. See the code for full details. + +### Why are variable values provided separately in `tfvars/` and not in the `yaml` file? -NOTE to Masterpoint team: We might want to create a small wrapper to automatize this using Taskit. On hold for now. +This is to support easy local and outside-spacelift operations. Keeping variable values in a `tfvars` file per workspace allows you to simply pass that file to the relevant CLI command locally via the `-var-file` option so that you don't need to provide values individually. e.g. `tofu plan -var-file=tfvars/dev.tfvars` diff --git a/examples/complete/components/random-pet/stacks/example.yaml b/examples/complete/components/random-pet/stacks/dev.yaml similarity index 100% rename from examples/complete/components/random-pet/stacks/example.yaml rename to examples/complete/components/random-pet/stacks/dev.yaml diff --git a/examples/complete/components/random-pet/tfvars/example.tfvars b/examples/complete/components/random-pet/tfvars/dev.tfvars similarity index 100% rename from examples/complete/components/random-pet/tfvars/example.tfvars rename to examples/complete/components/random-pet/tfvars/dev.tfvars diff --git a/examples/complete/components/spacelift-automation/backend.tf.json b/examples/complete/components/spacelift-automation/backend.tf.json deleted file mode 100644 index e4e1f6b..0000000 --- a/examples/complete/components/spacelift-automation/backend.tf.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "terraform": { - "backend": { - "s3": { - "workspace_key_prefix": "terraform-spacelift-automation", - "acl": "bucket-owner-full-control", - "bucket": "mp-automation-tfstate", - "dynamodb_table": "mp-automation-tfstate-lock", - "encrypt": true, - "key": "terraform.tfstate", - "region": "us-east-1", - "assume_role": { - "role_arn": "arn:aws:iam::755965222190:role/mp-automation-tfstate" - } - } - } - } -} diff --git a/examples/complete/components/spacelift-automation/main.tf b/examples/complete/components/spacelift-automation/main.tf index 1ca0be6..e932c76 100644 --- a/examples/complete/components/spacelift-automation/main.tf +++ b/examples/complete/components/spacelift-automation/main.tf @@ -8,5 +8,6 @@ module "automation" { root_modules_path = var.root_modules_path all_root_modules_enabled = var.all_root_modules_enabled - aws_integration_id = var.aws_integration_id + aws_integration_id = var.aws_integration_id + aws_integration_enabled = true } diff --git a/examples/complete/components/spacelift-automation/stacks/common.yaml b/examples/complete/components/spacelift-automation/stacks/common.yaml index 71b1ba6..7215bdb 100644 --- a/examples/complete/components/spacelift-automation/stacks/common.yaml +++ b/examples/complete/components/spacelift-automation/stacks/common.yaml @@ -1,5 +1,4 @@ stack_settings: administrative: true - aws_integration_enabled: true labels: - common_label diff --git a/examples/complete/components/spacelift-automation/tfvars/example.tfvars b/examples/complete/components/spacelift-automation/tfvars/example.tfvars index ce38dd7..4163048 100644 --- a/examples/complete/components/spacelift-automation/tfvars/example.tfvars +++ b/examples/complete/components/spacelift-automation/tfvars/example.tfvars @@ -4,4 +4,4 @@ github_enterprise = { repository = "terraform-spacelift-automation" root_modules_path = "../../../../examples/complete/components" all_root_modules_enabled = true -aws_integration_id = "01J30JBKQTCD72ATZCRWHYST3C" +aws_integration_id = "01JEC7ZACVKHTSVY4NF8QNZVVB" diff --git a/examples/single-instance/root-modules/random-pet/default.auto.tfvars b/examples/single-instance/root-modules/random-pet/default.auto.tfvars new file mode 100644 index 0000000..983ef02 --- /dev/null +++ b/examples/single-instance/root-modules/random-pet/default.auto.tfvars @@ -0,0 +1 @@ +length = 10 diff --git a/examples/single-instance/root-modules/random-pet/main.tf b/examples/single-instance/root-modules/random-pet/main.tf new file mode 100644 index 0000000..74e76fd --- /dev/null +++ b/examples/single-instance/root-modules/random-pet/main.tf @@ -0,0 +1,3 @@ +resource "random_pet" "template" { + length = var.length +} diff --git a/examples/single-instance/root-modules/random-pet/stack.yaml b/examples/single-instance/root-modules/random-pet/stack.yaml new file mode 100644 index 0000000..44f9498 --- /dev/null +++ b/examples/single-instance/root-modules/random-pet/stack.yaml @@ -0,0 +1,7 @@ +stack_settings: + manage_state: true + description: This stack generates random pet names + labels: + - common_label + - stack_specific_label +default_tf_workspace_enabled: true diff --git a/examples/single-instance/root-modules/random-pet/variables.tf b/examples/single-instance/root-modules/random-pet/variables.tf new file mode 100644 index 0000000..e44654e --- /dev/null +++ b/examples/single-instance/root-modules/random-pet/variables.tf @@ -0,0 +1,4 @@ +variable "length" { + description = "The length of the random name" + type = number +} diff --git a/examples/single-instance/root-modules/random-pet/versions.tf b/examples/single-instance/root-modules/random-pet/versions.tf new file mode 100644 index 0000000..e6f2511 --- /dev/null +++ b/examples/single-instance/root-modules/random-pet/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/examples/single-instance/root-modules/rds-cluster-dev/main.tf b/examples/single-instance/root-modules/rds-cluster-dev/main.tf new file mode 100644 index 0000000..5817a7b --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-dev/main.tf @@ -0,0 +1,5 @@ +resource "null_resource" "example" { + triggers = { + timestamp = timestamp() + } +} diff --git a/examples/single-instance/root-modules/rds-cluster-dev/stack.yaml b/examples/single-instance/root-modules/rds-cluster-dev/stack.yaml new file mode 100644 index 0000000..bffd033 --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-dev/stack.yaml @@ -0,0 +1,2 @@ +stack_settings: + description: This is a mock root module for Dev diff --git a/examples/single-instance/root-modules/rds-cluster-dev/versions.tf b/examples/single-instance/root-modules/rds-cluster-dev/versions.tf new file mode 100644 index 0000000..b1b0cc3 --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-dev/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0.0" + + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/examples/single-instance/root-modules/rds-cluster-prod/main.tf b/examples/single-instance/root-modules/rds-cluster-prod/main.tf new file mode 100644 index 0000000..5817a7b --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-prod/main.tf @@ -0,0 +1,5 @@ +resource "null_resource" "example" { + triggers = { + timestamp = timestamp() + } +} diff --git a/examples/single-instance/root-modules/rds-cluster-prod/stack.yaml b/examples/single-instance/root-modules/rds-cluster-prod/stack.yaml new file mode 100644 index 0000000..5b6c274 --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-prod/stack.yaml @@ -0,0 +1,2 @@ +stack_settings: + description: This is a mock root module for Prod diff --git a/examples/single-instance/root-modules/rds-cluster-prod/versions.tf b/examples/single-instance/root-modules/rds-cluster-prod/versions.tf new file mode 100644 index 0000000..b1b0cc3 --- /dev/null +++ b/examples/single-instance/root-modules/rds-cluster-prod/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0.0" + + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } +} diff --git a/examples/single-instance/root-modules/spacelift-automation-example2/main.tf b/examples/single-instance/root-modules/spacelift-automation-example2/main.tf new file mode 100644 index 0000000..dc2077f --- /dev/null +++ b/examples/single-instance/root-modules/spacelift-automation-example2/main.tf @@ -0,0 +1,15 @@ +module "automation" { + source = "../../../../" + + github_enterprise = { + namespace = "masterpointio" + } + repository = "terraform-spacelift-automation" + root_modules_path = "../../../../examples/single-instance/root-modules" + all_root_modules_enabled = true + + aws_integration_id = "01JEC7ZACVKHTSVY4NF8QNZVVB" + aws_integration_enabled = true + + root_module_structure = "SingleInstance" +} diff --git a/examples/single-instance/root-modules/spacelift-automation-example2/stack.yaml b/examples/single-instance/root-modules/spacelift-automation-example2/stack.yaml new file mode 100644 index 0000000..905291f --- /dev/null +++ b/examples/single-instance/root-modules/spacelift-automation-example2/stack.yaml @@ -0,0 +1,8 @@ +stack_settings: + description: This Automation stack is used for Masterpoint's testing purposes + administrative: true + aws_integration_enabled: false + labels: + - common_label + - stack_specific_label + default_tf_workspace_enabled: true diff --git a/examples/single-instance/root-modules/spacelift-automation-example2/versions.tf b/examples/single-instance/root-modules/spacelift-automation-example2/versions.tf new file mode 100644 index 0000000..a59e42a --- /dev/null +++ b/examples/single-instance/root-modules/spacelift-automation-example2/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0" + + required_providers { + spacelift = { + source = "spacelift-io/spacelift" + version = "~> 1.14" + } + } +} diff --git a/main.tf b/main.tf index 3426795..62aceab 100644 --- a/main.tf +++ b/main.tf @@ -25,8 +25,34 @@ # that are not directly used in the resource creation. locals { - # Read all stack files following the convention of root-module-name/stacks/*.yaml - _all_stack_files = fileset("${path.root}/${var.root_modules_path}/*/stacks", "*.yaml") + _multi_instance_structure = var.root_module_structure == "MultiInstance" + + # Read all stack files following the associated root_module_structue convention: + # MultiInstance: root-module-name/stacks/*.yaml + # SingleInstance: root-module-name/stack.yaml + # Example: + # [ + # MultiInstance: + # "../root-module-a/stacks/example.yaml", + # "../root-module-a/stacks/common.yaml", + # "../root-module-b/stacks/example.yaml", + # "../root-module-b/stacks/common.yaml", + # ] OR [ + # SingleInstance: + # "../root-module-a/stack.yaml", + # "../root-module-b/stack.yaml", + # ] + # "../root-module-a/stacks/example.yaml", + # "../root-module-a/stacks/common.yaml", + # "../root-module-b/stacks/example.yaml", + # "../root-module-b/stacks/common.yaml", + # ] OR [ + # "../root-module-a/stack.yaml", + # "../root-module-b/stack.yaml", + # ] + _multi_instance_stack_files = fileset("${path.root}/${var.root_modules_path}/*/stacks", "*.yaml") + _single_instance_stack_files = fileset("${path.root}/${var.root_modules_path}/*", "stack.yaml") + _all_stack_files = local._multi_instance_structure ? local._multi_instance_stack_files : local._single_instance_stack_files # Extract the root module name from the stack file path _all_root_modules = distinct([for file in local._all_stack_files : dirname(replace(replace(file, "../", ""), "stacks/", ""))]) @@ -36,37 +62,41 @@ locals { # Read and decode Stack YAML files from the root directory # Example: - # { + # MultiInstance: { # "random-pet" = { # "common.yaml" = { - # "stack_settings" = { - # "description" = "This stack generates random pet names" - # "manage_state" = true - # } - # "tfvars" = { - # "enabled" = false - # } + # "stack_settings" = { ... } + # ... # } # "example.yaml" = { - # "stack_settings" = { - # "manage_state" = true - # } - # "tfvars" = { - # "enabled" = true - # } + # "stack_settings" = { ... } + # ... # } # } # } - - _root_module_yaml_decoded = { + # SingleInstance: { + # "random-pet" = { + # "default" = { stack_settings = { ... }, ... } + # } + # } + _multi_instance_root_module_yaml_decoded = { for module in local.enabled_root_modules : module => { for yaml_file in fileset("${path.root}/${var.root_modules_path}/${module}/stacks", "*.yaml") : yaml_file => yamldecode(file("${path.root}/${var.root_modules_path}/${module}/stacks/${yaml_file}")) - } + } if local._multi_instance_structure + } + + _single_instance_root_module_yaml_decoded = { + for module in local.enabled_root_modules : module => { + "default" = yamldecode(file("${path.root}/${var.root_modules_path}/${module}/stack.yaml")) + } if !local._multi_instance_structure } + _root_module_yaml_decoded = merge(local._multi_instance_root_module_yaml_decoded, local._single_instance_root_module_yaml_decoded) + ## Common Stack configurations - # Retrieve common Stack configurations for each root module + # Retrieve common Stack configurations for each root module. + # SingleInstance root_module_structure does not support common configs today. # Example: # { # "random-pet" = { @@ -79,11 +109,13 @@ locals { # } # } # } - _common_configs = { for module, files in local._root_module_yaml_decoded : module => lookup(files, var.common_config_file, {}) } + # If we're SingleInstance, then default_tf_workspace_enabled is true. Otherwise, use given value. + _default_tf_workspace_enabled = local._multi_instance_structure ? var.default_tf_workspace_enabled : true + ## Stack Configurations # Merge all Stack configurations from the root modules into a single map, and filter out the common config. # Example: @@ -100,17 +132,18 @@ locals { # } # } # } - _root_module_stack_configs = merge([for module, files in local._root_module_yaml_decoded : { - for file, content in files : "${module}-${trimsuffix(file, ".yaml")}" => + for file, content in files : + local._multi_instance_structure ? "${module}-${trimsuffix(file, ".yaml")}" : module => merge( { "project_root" = replace(format("%s/%s", var.root_modules_path, module), "../", "") "root_module" = module, # If default_tf_workspace_enabled is true, use "default" workspace, otherwise our file name is the workspace name - "terraform_workspace" = try(content.default_tf_workspace_enabled, var.default_tf_workspace_enabled) ? "default" : trimsuffix(file, ".yaml"), + "terraform_workspace" = try(content.default_tf_workspace_enabled, local._default_tf_workspace_enabled) ? "default" : trimsuffix(file, ".yaml"), + # tfvars_file_name only pertains to MultiInstance, as SingleInstance expects consumers to use an auto.tfvars file. # `yaml` is intentionally used here as we require Stack and `tfvars` config files to be named equally "tfvars_file_name" = trimsuffix(file, ".yaml"), }, @@ -175,7 +208,6 @@ locals { # "depends-on:spacelift-automation-default", # ] # } - _dependency_labels = { for stack in local.stacks : stack => [ "depends-on:spacelift-automation-${terraform.workspace}" @@ -190,10 +222,9 @@ locals { # "folder:random-pet/example", # ] # } - _folder_labels = { for stack in local.stacks : stack => [ - "folder:${local.configs[stack].root_module}/${local.configs[stack].tfvars_file_name}" + local._multi_instance_structure ? "folder:${local.configs[stack].root_module}/${local.configs[stack].tfvars_file_name}" : "folder:${local.configs[stack].root_module}" ] } @@ -205,7 +236,6 @@ locals { # "depends-on:spacelift-automation-default", # ]) # } - labels = { for stack in local.stacks : stack => compact(flatten([ @@ -219,8 +249,9 @@ locals { # Merge all before_init steps into a single map for each stack. before_init = { for stack in local.stacks : stack => - # tfvars are implicitly enabled, which means we include the tfvars copy command in before_init - try(local.configs[stack].tfvars.enabled, true) ? + # tfvars are implicitly enabled in MultiInstance, which means we include the tfvars copy command in before_init + # In SingleInstance, we expect the consumer to use an auto.tfvars file, so we don't include the tfvars copy command in before_init + try(local.configs[stack].tfvars.enabled, local._multi_instance_structure) ? compact(concat( var.before_init, try(local.stack_configs[stack].before_init, []), @@ -288,21 +319,6 @@ resource "spacelift_stack" "default" { id = github_enterprise.value["id"] } } - - lifecycle { - # Expected `tfvars` file exists - precondition { - condition = try(local.configs[each.key].tfvars.enabled, true) ? fileexists("${local.configs[each.key].project_root}/tfvars/${local.configs[each.key].tfvars_file_name}.tfvars") : true - error_message = <<-EOT - The required .tfvars file is missing for stack "${each.key}". - - Expected location: - "${local.configs[each.key].project_root}/tfvars/${local.configs[each.key].tfvars_file_name}.tfvars" - - Ensure that the specified .tfvars file exists in the expected path and try again. - EOT - } - } } # The Spacelift Destructor is a feature designed to automatically clean up the resources no longer managed by our IaC. diff --git a/tests/fixtures/root-module-a/stacks/common.yaml b/tests/fixtures/multi-instance/root-module-a/stacks/common.yaml similarity index 100% rename from tests/fixtures/root-module-a/stacks/common.yaml rename to tests/fixtures/multi-instance/root-module-a/stacks/common.yaml diff --git a/tests/fixtures/root-module-a/stacks/default-example.yaml b/tests/fixtures/multi-instance/root-module-a/stacks/default-example.yaml similarity index 100% rename from tests/fixtures/root-module-a/stacks/default-example.yaml rename to tests/fixtures/multi-instance/root-module-a/stacks/default-example.yaml diff --git a/tests/fixtures/root-module-a/stacks/test.yaml b/tests/fixtures/multi-instance/root-module-a/stacks/test.yaml similarity index 100% rename from tests/fixtures/root-module-a/stacks/test.yaml rename to tests/fixtures/multi-instance/root-module-a/stacks/test.yaml diff --git a/tests/fixtures/root-module-a/tfvars/default-example.tfvars b/tests/fixtures/multi-instance/root-module-a/tfvars/default-example.tfvars similarity index 100% rename from tests/fixtures/root-module-a/tfvars/default-example.tfvars rename to tests/fixtures/multi-instance/root-module-a/tfvars/default-example.tfvars diff --git a/tests/fixtures/root-module-a/tfvars/test.tfvars b/tests/fixtures/multi-instance/root-module-a/tfvars/test.tfvars similarity index 100% rename from tests/fixtures/root-module-a/tfvars/test.tfvars rename to tests/fixtures/multi-instance/root-module-a/tfvars/test.tfvars diff --git a/tests/fixtures/single-instance/root-module-a/stack.yaml b/tests/fixtures/single-instance/root-module-a/stack.yaml new file mode 100644 index 0000000..9b9a5dd --- /dev/null +++ b/tests/fixtures/single-instance/root-module-a/stack.yaml @@ -0,0 +1,6 @@ +stack_settings: + administrative: false + before_init: + - echo 'World' + labels: + - stack_label diff --git a/tests/main.tftest.hcl b/tests/main.tftest.hcl index b821d27..a73d81d 100644 --- a/tests/main.tftest.hcl +++ b/tests/main.tftest.hcl @@ -1,9 +1,5 @@ -provider "spacelift" { - api_key_endpoint = "https://masterpointio.app.spacelift.io" -} - variables { - root_modules_path = "./tests/fixtures" + root_modules_path = "./tests/fixtures/multi-instance" common_config_file = "common.yaml" github_enterprise = { namespace = "masterpointio" diff --git a/tests/single-instance.tftest.hcl b/tests/single-instance.tftest.hcl new file mode 100644 index 0000000..b343ab3 --- /dev/null +++ b/tests/single-instance.tftest.hcl @@ -0,0 +1,130 @@ +variables { + root_modules_path = "./tests/fixtures/single-instance" + github_enterprise = { + namespace = "masterpointio" + } + repository = "terraform-spacelift-automation" + all_root_modules_enabled = true + aws_integration_enabled = false + before_init = [ + "echo 'Hello'" + ] + + root_module_structure = "SingleInstance" +} + +# Test that the root module fileset is created correctly +run "test_single_instance_root_module_fileset_collects_all_root_modules" { + command = plan + + assert { + condition = contains(local._all_root_modules, "root-module-a") + error_message = "Root module fileset was not created correctly: ${jsonencode(local._all_root_modules)}" + } +} + +# Test that the stack names are created correctly +run "test_single_instance_stacks_include_expected" { + command = plan + + assert { + condition = contains(local.stacks, "root-module-a") + error_message = "Stack names were not created correctly: ${jsonencode(local.stacks)}" + } +} + +run "test_single_instance_stacks_only_include_default_stack" { + command = plan + + assert { + condition = length(local._root_module_yaml_decoded["root-module-a"]) == 1 && local._root_module_yaml_decoded["root-module-a"]["default"] != null + error_message = "_root_module_yaml_decoded is not a single instance: ${jsonencode(local._root_module_yaml_decoded)}" + } +} + +run "test_single_instance_stack_configs_stack_name_is_correct" { + command = plan + + assert { + condition = length(local._root_module_stack_configs) == 1 && local._root_module_stack_configs["root-module-a"] != null + error_message = "_root_module_stack_configs is not expected structure: ${jsonencode(local._root_module_stack_configs)}" + } +} + +run "test_single_instance_stack_configs_use_default_tf_workspace" { + command = plan + + assert { + condition = local._root_module_stack_configs["root-module-a"].terraform_workspace == "default" + error_message = "terraform_workspace is not set to default: ${jsonencode(local._root_module_stack_configs)}" + } +} + +run "test_single_instance_stack_configs_project_root_is_correct" { + command = plan + + assert { + condition = local._root_module_stack_configs["root-module-a"].project_root == "${var.root_modules_path}/root-module-a" + error_message = "project_root is not correct for root-module-a: ${jsonencode(local._root_module_stack_configs)}" + } +} + +# Test that the administrative label is not added to the stack when the stack is not set to administrative +run "test_single_instance_administrative_label_is_not_added_to_stack_when_not_administrative" { + command = plan + + assert { + condition = !contains(local.labels["root-module-a"], "administrative") + error_message = "Administrative label was added to the stack when it should not have been: ${jsonencode(local.labels)}" + } +} + +# Test that the depends-on label is added to the stack +run "test_single_instance_depends_on_label_is_added_to_stack" { + command = plan + + assert { + condition = contains(local.labels["root-module-a"], "depends-on:spacelift-automation-default") + error_message = "Depends-on label was not added to the stack: ${jsonencode(local.labels)}" + } +} + +# Test that the folder label is added to the stack and doesn't include a workspace name +run "test_single_instance_folder_label_is_added_to_stack_and_doesnt_include_workspace_name" { + command = plan + + assert { + condition = contains(local.labels["root-module-a"], "folder:root-module-a") + error_message = "Folder label was not added to the stack: ${jsonencode(local.labels)}" + } +} + +# Test that stack.yaml labels are included in the stack labels +run "test_single_instance_stack_yaml_labels_are_included_in_stack_labels" { + command = plan + + assert { + condition = contains(local.labels["root-module-a"], "stack_label") + error_message = "Stack.yaml labels were not included in the stack labels: ${jsonencode(local.labels)}" + } +} + +# Test that the before_init steps are added to the stack +run "test_single_instance_before_init_steps_are_added_to_stack" { + command = plan + + assert { + condition = contains(local.before_init["root-module-a"], "echo 'Hello'") && contains(local.before_init["root-module-a"], "echo 'World'") + error_message = "Before_init steps were not added to the stack: ${jsonencode(local.before_init)}" + } +} + +# Test that the before_init tfvar cp command is not added to the stack +run "test_single_instance_before_init_tfvar_cp_command_is_not_added_to_stack" { + command = plan + + assert { + condition = !contains(local.before_init["root-module-a"], "cp tfvars/") + error_message = "Before_init tfvar cp command was added to the stack: ${jsonencode(local.before_init)}" + } +} diff --git a/variables.tf b/variables.tf index ddcb2fd..b2898e9 100644 --- a/variables.tf +++ b/variables.tf @@ -1,4 +1,19 @@ -# GitHub +variable "root_module_structure" { + type = string + description = <<-EOT + The root module structure of the Stacks that you're reading in. See README for full details. + + MultiInstance - You're using Workspaces or Dynamic Backend configuration to create multiple instances of the same root module code. + SingleInstance - You're using copies of a root module and your directory structure to create multiple instances of the same Terraform code. + EOT + default = "MultiInstance" + + validation { + condition = contains(["MultiInstance", "SingleInstance"], var.root_module_structure) + error_message = "Valid values for root_module_structure are (MultiInstance, SingleInstance)." + } +} + variable "github_enterprise" { type = object({ namespace = string @@ -254,7 +269,7 @@ variable "protect_from_deletion" { variable "space_id" { type = string - description = "Place the stack in the specified space_id." + description = "Place the created stacks in the specified space_id." default = "root" }