Vault (#201)

* Vault permissions

* Enable Vault initialization

* Add Vault address

* GitLab variables

* Use GitLab group slug for Vault

* subrepo Vault support

* Add regcred secrets to Vault

* Terraform Cloud secret role

* Update requirements

* Update pipeline with gitlab variables

* Add JWT auth to Vault TF

* Fetch TFC token from Vault

* Fetch Vault secrets

* Refactor deploy scripts

* Refactor Terraform and enhance failures handling

* Pact secrets support in Vault admin TF

* Pact Vault secret creation

* Pact Vault policy

Co-authored-by: Filippo Morelli <filippo@20tab.com>

.pre-commit-config.yaml

@@ -26,12 +26,12 @@ repos:
       - id: mixed-line-ending
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: "v2.31.1"
+    rev: "v2.34.0"
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
   - repo: https://github.com/psf/black
-    rev: "22.1.0"
+    rev: "22.3.0"
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
@@ -48,6 +48,6 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies:
-          - flake8-bugbear~=22.1.11
+          - flake8-bugbear~=22.4.25
           - flake8-docstrings~=1.6.0
           - flake8-isort~=4.1.0

Makefile

@@ -30,7 +30,7 @@ pip: pip_update  ## Compile requirements
 
 .PHONY: pip_update
 pip_update:  ## Update requirements and dependencies
-	python3 -m pip install -q -U pip~=22.0.0 pip-tools~=6.5.0 setuptools~=60.10.0 wheel~=0.37.0
+	python3 -m pip install -q -U pip~=22.1.0 pip-tools~=6.6.0 setuptools~=60.10.0 wheel~=0.37.0
 
 .PHONY: precommit
 precommit:  ## Fix code formatting, linting and sorting imports

README.md

@@ -195,10 +195,10 @@ The following arguments can be appended to the Docker and shell commands
 
 #### Frontend type
 
-| Value  | Description                                           | Argument                 |
-| ------ | ----------------------------------------------------- | ------------------------ |
+| Value  | Description                                         | Argument                 |
+| ------ | --------------------------------------------------- | ------------------------ |
 | nextjs | https://github.com/20tab/nextjs-continuous-delivery | `--frontend-type=nextjs` |
-| none   | the frontend service will not be initialized          | `--frontend-type=none`   |
+| none   | the frontend service will not be initialized        | `--frontend-type=none`   |
 
 #### Frontend service slug
 

bootstrap/collector.py

@@ -26,6 +26,7 @@
     ENVIRONMENT_DISTRIBUTION_PROMPT,
     FRONTEND_TYPE_CHOICES,
     FRONTEND_TYPE_DEFAULT,
+    GITLAB_URL_DEFAULT,
     MEDIA_STORAGE_AWS_S3,
     MEDIA_STORAGE_CHOICES,
     MEDIA_STORAGE_DIGITALOCEAN_S3,
@@ -58,6 +59,8 @@ def collect(
     terraform_cloud_organization,
     terraform_cloud_organization_create,
     terraform_cloud_admin_email,
+    vault_token,
+    vault_url,
     digitalocean_token,
     kubernetes_cluster_ca_certificate,
     kubernetes_host,
@@ -99,6 +102,7 @@ def collect(
     s3_access_id,
     s3_secret_key,
     s3_bucket_name,
+    gitlab_url,
     gitlab_private_token,
     gitlab_group_slug,
     gitlab_group_owners,
@@ -144,6 +148,7 @@ def collect(
         terraform_cloud_organization_create,
         terraform_cloud_admin_email,
     )
+    vault_token, vault_url = clean_vault_data(vault_token, vault_url, quiet)
     environment_distribution = clean_environment_distribution(
         environment_distribution, deployment_type
     )
@@ -242,22 +247,23 @@ def collect(
         pact_broker_url, pact_broker_username, pact_broker_password
     )
     (
+        gitlab_url,
         gitlab_group_slug,
         gitlab_private_token,
         gitlab_group_owners,
         gitlab_group_maintainers,
         gitlab_group_developers,
     ) = clean_gitlab_group_data(
         project_slug,
+        gitlab_url,
         gitlab_group_slug,
         gitlab_private_token,
         gitlab_group_owners,
         gitlab_group_maintainers,
         gitlab_group_developers,
         quiet,
     )
-    # TODO: change when moving secrets to Vault
-    if gitlab_group_slug and "s3" in media_storage:
+    if (gitlab_url or vault_url) and "s3" in media_storage:
         (
             digitalocean_token,
             s3_region,
@@ -295,6 +301,8 @@ def collect(
         "terraform_cloud_organization": terraform_cloud_organization,
         "terraform_cloud_organization_create": terraform_cloud_organization_create,
         "terraform_cloud_admin_email": terraform_cloud_admin_email,
+        "vault_token": vault_token,
+        "vault_url": vault_url,
         "digitalocean_token": digitalocean_token,
         "kubernetes_cluster_ca_certificate": kubernetes_cluster_ca_certificate,
         "kubernetes_host": kubernetes_host,
@@ -340,6 +348,7 @@ def collect(
         "s3_access_id": s3_access_id,
         "s3_secret_key": s3_secret_key,
         "s3_bucket_name": s3_bucket_name,
+        "gitlab_url": gitlab_url,
         "gitlab_private_token": gitlab_private_token,
         "gitlab_group_slug": gitlab_group_slug,
         "gitlab_group_owners": gitlab_group_owners,
@@ -491,7 +500,7 @@ def clean_terraform_backend(
     terraform_cloud_organization_create,
     terraform_cloud_admin_email,
 ):
-    """Return the terraform backend and the Terraform Cloud data, if applicable."""
+    """Return the Terraform backend and the Terraform Cloud data, if applicable."""
     terraform_backend = (
         terraform_backend
         if terraform_backend in TERRAFORM_BACKEND_CHOICES
@@ -545,6 +554,29 @@ def clean_terraform_backend(
     )
 
 
+def clean_vault_data(vault_token, vault_url, quiet=False):
+    """Return the Vault data, if applicable."""
+    if vault_token or (
+        vault_token is None
+        and click.confirm(
+            "Do you want to use Vault for secrets management?",
+        )
+    ):
+        vault_token = validate_or_prompt_password("Vault token", vault_token)
+        quiet or click.confirm(
+            warning(
+                "Make sure the Vault token has enough permissions to enable the "
+                "project secrets backends and manage the project secrets. Continue?"
+            ),
+            abort=True,
+        )
+        vault_url = validate_or_prompt_url("Vault address", vault_url)
+    else:
+        vault_token = None
+        vault_url = None
+    return vault_token, vault_url
+
+
 def clean_environment_distribution(environment_distribution, deployment_type):
     """Return the environment distribution."""
     if deployment_type == DEPLOYMENT_TYPE_OTHER:
@@ -864,6 +896,7 @@ def clean_media_storage(media_storage):
 
 def clean_gitlab_group_data(
     project_slug,
+    gitlab_url,
     gitlab_group_slug,
     gitlab_private_token,
     gitlab_group_owners,
@@ -872,10 +905,13 @@ def clean_gitlab_group_data(
     quiet=False,
 ):
     """Return GitLab group data."""
-    if gitlab_group_slug or (
-        gitlab_group_slug is None
+    if gitlab_url or (
+        gitlab_url is None
         and click.confirm(warning("Do you want to use GitLab?"), default=True)
     ):
+        gitlab_url = validate_or_prompt_url(
+            "GitLab URL", gitlab_url, default=GITLAB_URL_DEFAULT
+        )
         gitlab_group_slug = slugify(
             gitlab_group_slug or click.prompt("GitLab group slug", default=project_slug)
         )
@@ -905,12 +941,14 @@ def clean_gitlab_group_data(
             else click.prompt("Comma-separated GitLab group developers", default="")
         )
     else:
+        gitlab_url = None
         gitlab_group_slug = None
         gitlab_private_token = None
         gitlab_group_owners = None
         gitlab_group_maintainers = None
         gitlab_group_developers = None
     return (
+        gitlab_url,
         gitlab_group_slug,
         gitlab_private_token,
         gitlab_group_owners,

bootstrap/constants.py

@@ -4,6 +4,28 @@
 
 DUMPS_DIR = Path(__file__).parent.parent / ".dumps"
 
+# Stacks
+
+DEV_STACK_SLUG = "dev"
+
+STAGE_STACK_SLUG = "stage"
+
+MAIN_STACK_SLUG = "main"
+
+# Environments
+
+DEV_ENV_NAME = "development"
+
+DEV_ENV_SLUG = "dev"
+
+STAGE_ENV_NAME = "staging"
+
+STAGE_ENV_SLUG = "stage"
+
+PROD_ENV_NAME = "production"
+
+PROD_ENV_SLUG = "prod"
+
 # Env vars
 
 GITLAB_TOKEN_ENV_VAR = "GITLAB_PRIVATE_TOKEN"
@@ -18,11 +40,11 @@
     "nextjs": "https://github.com/20tab/nextjs-continuous-delivery"
 }
 
-SUBREPOS_DIR = ".subrepos"
+SUBREPOS_DIR = Path(__file__).parent.parent / ".subrepos"
 
 # Services type
 
-ORCHESTRATOR_SERVICE_SLUG = "orchestrator"
+SERVICE_SLUG_DEFAULT = "orchestrator"
 
 EMPTY_SERVICE_TYPE = "none"
 
@@ -86,3 +108,7 @@
 TERRAFORM_BACKEND_TFC = "terraform-cloud"
 
 TERRAFORM_BACKEND_CHOICES = [TERRAFORM_BACKEND_TFC, TERRAFORM_BACKEND_GITLAB]
+
+# GitLab
+
+GITLAB_URL_DEFAULT = "https://gitlab.com"

bootstrap/helpers.py

@@ -3,6 +3,16 @@
 from slugify import slugify
 
 
+def format_gitlab_variable(value, masked=False, protected=True):
+    """Format the given value to be used as a Terraform variable."""
+    return (
+        f'{{ value = "{value}"'
+        + (masked and ", masked = true" or "")
+        + (not protected and ", protected = false" or "")
+        + "}"
+    )
+
+
 def format_tfvar(value, value_type=None):
     """Format the given value to be used as a Terraform variable."""
     if value_type == "list":