From 12901ae5eef1046cb1de4833b6773e65b31edf7d Mon Sep 17 00:00:00 2001 From: Filippo Morelli Date: Wed, 7 Dec 2022 16:13:55 +0100 Subject: [PATCH] Refactor Vault resources (#210) * Refactor Vault resources Co-authored-by: Daniele Pompa <55095241+daniele20tab@users.noreply.github.com> --- bootstrap/collector.py | 105 +++++----- bootstrap/constants.py | 37 ++++ bootstrap/runner.py | 198 ++++++++++-------- cookiecutter.json | 60 ++++-- start.py | 3 +- terraform/gitlab/main.tf | 20 +- terraform/gitlab/outputs.tf | 5 - terraform/terraform-cloud/variables.tf | 2 + terraform/vault-admin/main.tf | 197 ----------------- terraform/vault-admin/variables.tf | 43 ---- terraform/vault/main.tf | 18 +- terraform/vault/variables.tf | 12 ++ tests/test_collector.py | 76 +++++-- .../.gitlab-ci.yml | 148 ++++++------- .../scripts/deploy/vault.sh | 31 +-- 15 files changed, 419 insertions(+), 536 deletions(-) delete mode 100644 terraform/vault-admin/main.tf delete mode 100644 terraform/vault-admin/variables.tf diff --git a/bootstrap/collector.py b/bootstrap/collector.py index f7938477..c84e3626 100755 --- a/bootstrap/collector.py +++ b/bootstrap/collector.py @@ -132,7 +132,7 @@ def collect( deployment_type = clean_deployment_type(deployment_type) # The "digitalocean-k8s" deployment type includes Postgres by default if digitalocean_enabled := ("digitalocean" in deployment_type): - digitalocean_token = validate_or_prompt_password( + digitalocean_token = validate_or_prompt_secret( "DigitalOcean token", digitalocean_token ) ( @@ -236,9 +236,9 @@ def collect( sentry_org, sentry_url, sentry_auth_token, - backend_type, + backend_service_slug, backend_sentry_dsn, - frontend_type, + frontend_service_slug, frontend_sentry_dsn, ) ( @@ -394,8 +394,8 @@ def validate_or_prompt_email(message, value=None, default=None, required=True): return validate_or_prompt_email(message, None, default, required) -def validate_or_prompt_password(message, value=None, default=None, required=True): - """Validate the given password or prompt until a valid value is provided.""" +def validate_or_prompt_secret(message, value=None, default=None, required=True): + """Validate the given secret or prompt until a valid value is provided.""" if value is None: value = click.prompt(message, default=default, hide_input=True) try: @@ -404,7 +404,7 @@ def validate_or_prompt_password(message, value=None, default=None, required=True except validators.ValidationFailure: pass click.echo(error("Please type at least 8 chars!")) - return validate_or_prompt_password(message, None, default, required) + return validate_or_prompt_secret(message, None, default, required) def validate_or_prompt_path(message, value=None, default=None, required=True): @@ -543,7 +543,7 @@ def clean_terraform_backend( terraform_cloud_hostname, default="app.terraform.io", ) - terraform_cloud_token = validate_or_prompt_password( + terraform_cloud_token = validate_or_prompt_secret( "Terraform Cloud User token", terraform_cloud_token, ) @@ -583,16 +583,21 @@ def clean_terraform_backend( def clean_vault_data(vault_token, vault_url, quiet=False): """Return the Vault data, if applicable.""" - if vault_token or ( - vault_token is None + if vault_url or ( + vault_url is None and click.confirm( "Do you want to use Vault for secrets management?", ) ): - vault_token = validate_or_prompt_password("Vault token", vault_token) + vault_token = validate_or_prompt_secret( + "Vault token (leave blank to perform a browser-based OIDC authentication)", + vault_token, + default="", + required=False, + ) quiet or click.confirm( warning( - "Make sure the Vault token has enough permissions to enable the " + "Make sure your Vault permissions allow to enable the " "project secrets backends and manage the project secrets. Continue?" ), abort=True, @@ -635,7 +640,7 @@ def clean_kubernetes_credentials( kubernetes_host = kubernetes_host or validate_or_prompt_url( "Kubernetes host", kubernetes_host ) - kubernetes_token = kubernetes_token or validate_or_prompt_password( + kubernetes_token = kubernetes_token or validate_or_prompt_secret( "Kubernetes token", kubernetes_token ) return kubernetes_cluster_ca_certificate, kubernetes_host, kubernetes_token @@ -710,30 +715,49 @@ def clean_letsencrypt_certificate_email(letsencrypt_certificate_email): ) +def clean_sentry_org(sentry_org): + """Return the Sentry organization.""" + return sentry_org if sentry_org is not None else click.prompt("Sentry organization") + + +def clean_sentry_dsn(service_slug, sentry_dsn): + """Return the backend Sentry DSN.""" + if service_slug: + return validate_or_prompt_url( + f"Sentry DSN of the {service_slug} service (leave blank if unused)", + sentry_dsn, + default="", + required=False, + ) + + def clean_sentry_data( sentry_org, sentry_url, sentry_auth_token, - backend_type, + backend_service_slug, backend_sentry_dsn, - frontend_type, + frontend_service_slug, frontend_sentry_dsn, ): """Return the Sentry configuration data.""" - if sentry_org or ( - sentry_org is None - and click.confirm(warning("Do you want to use Sentry?"), default=False) + if any((backend_service_slug, frontend_service_slug)) and ( + sentry_org + or ( + sentry_org is None + and click.confirm(warning("Do you want to use Sentry?"), default=False) + ) ): sentry_org = clean_sentry_org(sentry_org) sentry_url = validate_or_prompt_url( "Sentry URL", sentry_url, default="https://sentry.io/" ) - sentry_auth_token = validate_or_prompt_password( + sentry_auth_token = validate_or_prompt_secret( "Sentry auth token", sentry_auth_token ) - backend_sentry_dsn = clean_backend_sentry_dsn(backend_type, backend_sentry_dsn) - frontend_sentry_dsn = clean_frontend_sentry_dsn( - frontend_type, frontend_sentry_dsn + backend_sentry_dsn = clean_sentry_dsn(backend_service_slug, backend_sentry_dsn) + frontend_sentry_dsn = clean_sentry_dsn( + frontend_service_slug, frontend_sentry_dsn ) else: sentry_org = None @@ -750,37 +774,6 @@ def clean_sentry_data( ) -def clean_sentry_org(sentry_org): - """Return the Sentry organization.""" - return sentry_org if sentry_org is not None else click.prompt("Sentry organization") - - -def clean_backend_sentry_dsn(backend_type, backend_sentry_dsn): - """Return the backend Sentry DSN.""" - if backend_type: - return ( - backend_sentry_dsn - if backend_sentry_dsn is not None - else click.prompt( - "Backend Sentry DSN (leave blank if unused)", - default="", - ) - ) - - -def clean_frontend_sentry_dsn(frontend_type, frontend_sentry_dsn): - """Return the frontend Sentry DSN.""" - if frontend_type: - return ( - frontend_sentry_dsn - if frontend_sentry_dsn is not None - else click.prompt( - "Frontend Sentry DSN (leave blank if unused)", - default="", - ) - ) - - def clean_digitalocean_options( digitalocean_domain_create, digitalocean_dns_records_create, @@ -899,7 +892,7 @@ def clean_pact_broker_data(pact_broker_url, pact_broker_username, pact_broker_pa pact_broker_username = pact_broker_username or click.prompt( "Pact broker username", ) - pact_broker_password = validate_or_prompt_password( + pact_broker_password = validate_or_prompt_secret( "Pact broker password", pact_broker_password ) else: @@ -1008,7 +1001,7 @@ def clean_s3_media_storage_data( ): """Return S3 media storage data.""" if media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3: - digitalocean_token = validate_or_prompt_password( + digitalocean_token = validate_or_prompt_secret( "DigitalOcean token", digitalocean_token ) s3_region = s3_region or click.prompt( @@ -1027,8 +1020,8 @@ def clean_s3_media_storage_data( s3_bucket_name = s3_bucket_name or click.prompt( "AWS S3 bucket name", ) - s3_access_id = validate_or_prompt_password("S3 Access Key ID", s3_access_id) - s3_secret_key = validate_or_prompt_password("S3 Secret Access Key", s3_secret_key) + s3_access_id = validate_or_prompt_secret("S3 Access Key ID", s3_access_id) + s3_secret_key = validate_or_prompt_secret("S3 Secret Access Key", s3_secret_key) return ( digitalocean_token, s3_region, diff --git a/bootstrap/constants.py b/bootstrap/constants.py index 4ce88803..981686fc 100755 --- a/bootstrap/constants.py +++ b/bootstrap/constants.py @@ -1,35 +1,72 @@ """Web project initialization CLI constants.""" from pathlib import Path +from typing import Dict DUMPS_DIR = Path(__file__).parent.parent / ".dumps" # Stacks +# BEWARE: stack names must be suitable for inclusion in Vault paths + +DEV_STACK_NAME = "development" + DEV_STACK_SLUG = "dev" +STAGE_STACK_NAME = "staging" + STAGE_STACK_SLUG = "stage" +MAIN_STACK_NAME = "main" + MAIN_STACK_SLUG = "main" +STACKS_CHOICES = { + "1": [{"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG}], + "2": [ + {"name": DEV_STACK_NAME, "slug": DEV_STACK_SLUG}, + {"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG}, + ], + "3": [ + {"name": DEV_STACK_NAME, "slug": DEV_STACK_SLUG}, + {"name": STAGE_STACK_NAME, "slug": STAGE_STACK_SLUG}, + {"name": MAIN_STACK_NAME, "slug": MAIN_STACK_SLUG}, + ], +} + # Environments +# BEWARE: environment names must be suitable for inclusion in Vault paths + DEV_ENV_NAME = "development" DEV_ENV_SLUG = "dev" +DEV_ENV_STACK_CHOICES: Dict[str, str] = { + "1": MAIN_STACK_SLUG, +} + STAGE_ENV_NAME = "staging" STAGE_ENV_SLUG = "stage" +STAGE_ENV_STACK_CHOICES: Dict[str, str] = { + "1": MAIN_STACK_SLUG, + "2": DEV_STACK_SLUG, +} + PROD_ENV_NAME = "production" PROD_ENV_SLUG = "prod" +PROD_ENV_STACK_CHOICES: Dict[str, str] = {} + # Env vars GITLAB_TOKEN_ENV_VAR = "GITLAB_PRIVATE_TOKEN" +VAULT_TOKEN_ENV_VAR = "VAULT_TOKEN" + # Subrepos BACKEND_TEMPLATE_URLS = { diff --git a/bootstrap/runner.py b/bootstrap/runner.py index ca4f176e..e0d637cd 100644 --- a/bootstrap/runner.py +++ b/bootstrap/runner.py @@ -9,6 +9,8 @@ import subprocess from dataclasses import dataclass, field from functools import partial +from itertools import groupby +from operator import itemgetter from pathlib import Path from time import time @@ -21,17 +23,22 @@ DEPLOYMENT_TYPE_OTHER, DEV_ENV_NAME, DEV_ENV_SLUG, + DEV_ENV_STACK_CHOICES, DEV_STACK_SLUG, DUMPS_DIR, FRONTEND_TEMPLATE_URLS, GITLAB_URL_DEFAULT, + MAIN_STACK_NAME, MAIN_STACK_SLUG, MEDIA_STORAGE_DIGITALOCEAN_S3, PROD_ENV_NAME, PROD_ENV_SLUG, + PROD_ENV_STACK_CHOICES, SERVICE_SLUG_DEFAULT, + STACKS_CHOICES, STAGE_ENV_NAME, STAGE_ENV_SLUG, + STAGE_ENV_STACK_CHOICES, STAGE_STACK_SLUG, SUBREPOS_DIR, TERRAFORM_BACKEND_TFC, @@ -127,7 +134,8 @@ class Runner: logs_dir: Path | None = None run_id: str = field(init=False) service_slug: str = field(init=False) - stacks_environments: dict = field(init=False, default_factory=dict) + stacks: list = field(init=False, default_factory=list) + envs: list = field(init=False, default_factory=list) gitlab_variables: dict = field(init=False, default_factory=dict) tfvars: dict = field(init=False, default_factory=dict) vault_secrets: dict = field(init=False, default_factory=dict) @@ -141,46 +149,49 @@ def __post_init__(self): self.run_id = f"{time():.0f}" self.terraform_dir = self.terraform_dir or Path(f".terraform/{self.run_id}") self.logs_dir = self.logs_dir or Path(f".logs/{self.run_id}") - self.set_stacks_environments() + self.set_stacks() + self.set_envs() self.collect_tfvars() self.collect_gitlab_variables() - def set_stacks_environments(self): - """Set the environments distribution per stack.""" - dev_env = { - "name": DEV_ENV_NAME, - "prefix": self.subdomain_dev, - "url": self.project_url_dev, - } - stage_env = { - "name": STAGE_ENV_NAME, - "prefix": self.subdomain_stage, - "url": self.project_url_stage, - } - prod_env = { - "name": PROD_ENV_NAME, - "prefix": self.subdomain_prod, - "url": self.project_url_prod, - } - if self.environment_distribution == "1": - self.stacks_environments = { - MAIN_STACK_SLUG: { - DEV_ENV_SLUG: dev_env, - STAGE_ENV_SLUG: stage_env, - PROD_ENV_SLUG: prod_env, - } - } - elif self.environment_distribution == "2": - self.stacks_environments = { - DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env, STAGE_ENV_SLUG: stage_env}, - MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env}, - } - elif self.environment_distribution == "3": - self.stacks_environments = { - DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env}, - STAGE_STACK_SLUG: {STAGE_ENV_SLUG: stage_env}, - MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env}, - } + def set_stacks(self): + """Set the stacks.""" + self.stacks = STACKS_CHOICES[self.environment_distribution] + + def set_envs(self): + """Set the envs.""" + self.envs = [ + { + "basic_auth_enabled": True, + "name": DEV_ENV_NAME, + "prefix": self.subdomain_dev, + "slug": DEV_ENV_SLUG, + "stack_slug": DEV_ENV_STACK_CHOICES.get( + self.environment_distribution, DEV_STACK_SLUG + ), + "url": self.project_url_dev, + }, + { + "basic_auth_enabled": True, + "name": STAGE_ENV_NAME, + "prefix": self.subdomain_stage, + "slug": STAGE_ENV_SLUG, + "stack_slug": STAGE_ENV_STACK_CHOICES.get( + self.environment_distribution, STAGE_STACK_SLUG + ), + "url": self.project_url_stage, + }, + { + "basic_auth_enabled": False, + "name": PROD_ENV_NAME, + "prefix": self.subdomain_prod, + "slug": PROD_ENV_SLUG, + "stack_slug": PROD_ENV_STACK_CHOICES.get( + self.environment_distribution, MAIN_STACK_SLUG + ), + "url": self.project_url_prod, + }, + ] def register_gitlab_variable( self, level, var_name, var_value=None, masked=False, protected=True @@ -375,27 +386,36 @@ def collect_tfvars(self): self.register_environment_tfvars( ("digitalocean_spaces_bucket_available", True, "bool") ) - for stack_slug, stack_envs in self.stacks_environments.items(): - for env_slug, _env_data in stack_envs.items(): - self.register_environment_tfvars( - ("basic_auth_enabled", env_slug != "prod", "bool"), - ("stack_slug", stack_slug), - ("subdomains", [getattr(self, f"subdomain_{env_slug}")], "list"), - env_slug=env_slug, - ) + for env in self.envs: + env_slug = env["slug"] + self.register_environment_tfvars( + ("basic_auth_enabled", env["basic_auth_enabled"], "bool"), + ("stack_slug", env["stack_slug"]), + ("subdomains", [getattr(self, f"subdomain_{env_slug}")], "list"), + env_slug=env_slug, + ) - def register_vault_stack_secret(self, stack_slug, name, data): - """Register a Vault stack secret locally.""" - self.vault_secrets[f"stacks/{stack_slug}/{name}"] = data + def register_vault_stack_secret( + self, stack_name, stack_envs_names, secret_name, secret_data + ): + """Register a Vault stack secret locally, optionally copying it to the envs.""" + self.vault_secrets[f"stacks/{stack_name}/{secret_name}"] = secret_data + [ + self.register_vault_environment_secret(i, secret_name, secret_data) + for i in stack_envs_names + ] - def register_vault_environment_secret(self, env_slug, name, data): + def register_vault_environment_secret(self, env_name, secret_name, secret_data): """Register a Vault environment secret locally.""" - self.vault_secrets[f"envs/{env_slug}/{name}"] = data + self.vault_secrets[f"envs/{env_name}/{secret_name}"] = secret_data - def collect_vault_stack_secrets(self, stack_slug): + def collect_vault_stack_secrets(self, stack_name, stack_envs_names): """Collect the Vault secrets for the given stack.""" self.digitalocean_token and self.register_vault_stack_secret( - stack_slug, "digitalocean", dict(digitalocean_token=self.digitalocean_token) + stack_name, + stack_envs_names, + "digitalocean", + dict(digitalocean_token=self.digitalocean_token), ) if "s3" in self.media_storage: s3_secret = dict( @@ -407,12 +427,15 @@ def collect_vault_stack_secrets(self, stack_slug): self.media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3 and s3_secret.update( s3_host=self.s3_host ) - self.register_vault_stack_secret(stack_slug, "s3", s3_secret) + self.register_vault_stack_secret( + stack_name, stack_envs_names, "s3", s3_secret + ) ( self.subdomain_monitoring - and stack_slug == MAIN_STACK_SLUG + and stack_name == MAIN_STACK_NAME and self.register_vault_stack_secret( - stack_slug, + stack_name, + stack_envs_names, "monitoring", dict(grafana_password=secrets.token_urlsafe(12)), ) @@ -420,7 +443,7 @@ def collect_vault_stack_secrets(self, stack_slug): ( self.deployment_type == DEPLOYMENT_TYPE_OTHER and self.register_vault_stack_secret( - stack_slug, + stack_name, "k8s", dict( kubernetes_cluster_ca_certificate=base64.b64encode( @@ -432,11 +455,11 @@ def collect_vault_stack_secrets(self, stack_slug): ) ) - def collect_vault_environment_secrets(self, env_slug): + def collect_vault_environment_secrets(self, env_name): """Collect the Vault secrets for the given environment.""" self.register_vault_environment_secret( - env_slug, - "basic_auth", + env_name, + f"{self.service_slug}/basic_auth", dict( basic_auth_username=self.project_slug, basic_auth_password=secrets.token_urlsafe(12), @@ -444,7 +467,7 @@ def collect_vault_environment_secrets(self, env_slug): ) # Sentry secrets are used by the GitLab CI/CD self.sentry_org and self.register_vault_environment_secret( - env_slug, "sentry", dict(sentry_auth_token=self.sentry_auth_token) + env_name, "sentry", dict(sentry_auth_token=self.sentry_auth_token) ) def collect_vault_pact_secrets(self): @@ -473,13 +496,18 @@ def collect_vault_secrets(self): registry_username=gitlab_terraform_outputs["registry_username"], registry_password=gitlab_terraform_outputs["registry_password"], ) - for stack_slug, stack_envs in self.stacks_environments.items(): - self.collect_vault_stack_secrets(stack_slug) - for env_slug in stack_envs: - self.collect_vault_environment_secrets(env_slug) + stacks_mapping = {i["slug"]: i["name"] for i in self.stacks} + for stack_slug, stack_envs in groupby(self.envs, key=itemgetter("stack_slug")): + stack_name = stacks_mapping[stack_slug] + stack_envs_names = [] + for env in stack_envs: + env_name = env["name"] + self.collect_vault_environment_secrets(env_name) regcred and self.register_vault_environment_secret( - env_slug, "regcred", regcred + env_name, f"{self.service_slug}/regcred", regcred ) + stack_envs_names.append(env_name) + self.collect_vault_stack_secrets(stack_name, stack_envs_names) self.pact_broker_url and self.collect_vault_pact_secrets() def init_service(self): @@ -500,8 +528,8 @@ def init_service(self): "project_dirname": self.project_dirname, "project_name": self.project_name, "project_slug": self.project_slug, + "resources": {"envs": self.envs, "stacks": self.stacks}, "service_slug": self.service_slug, - "stacks": self.stacks_environments, "terraform_backend": self.terraform_backend, "terraform_cloud_organization": self.terraform_cloud_organization, "tfvars": self.tfvars, @@ -548,16 +576,7 @@ def init_gitlab(self): GITLAB_BASE_URL=f"{self.gitlab_url}/api/v4/" ) self.run_terraform( - "gitlab", - env, - outputs=["registry_password", "registry_username", "ssh_url_to_repo"], - ) - self.make_sed( - "README.md", - "__VCS_BASE_SSH_URL__", - self.terraform_outputs["gitlab"]["ssh_url_to_repo"] - .replace(f"/{self.service_slug}.git", "") - .replace("/", "\\/"), + "gitlab", env, outputs=["registry_password", "registry_username"] ) def init_terraform_cloud(self): @@ -568,15 +587,13 @@ def init_terraform_cloud(self): TF_VAR_create_organization=self.terraform_cloud_organization_create and "true" or "false", - TF_VAR_environments=json.dumps( - [i for j in self.stacks_environments.values() for i in j] - ), + TF_VAR_environments=json.dumps(list(map(itemgetter("slug"), self.envs))), TF_VAR_hostname=self.terraform_cloud_hostname, TF_VAR_organization_name=self.terraform_cloud_organization, TF_VAR_project_name=self.project_name, TF_VAR_project_slug=self.project_slug, TF_VAR_service_slug=self.service_slug, - TF_VAR_stacks=json.dumps(list(self.stacks_environments)), + TF_VAR_stacks=json.dumps(list(map(itemgetter("slug"), self.stacks))), TF_VAR_terraform_cloud_token=self.terraform_cloud_token, ) self.run_terraform("terraform-cloud", env) @@ -590,8 +607,8 @@ def init_vault(self): TF_VAR_project_name=self.project_name, TF_VAR_project_slug=self.project_slug, TF_VAR_secrets=json.dumps(self.vault_secrets), - VAULT_ADDR=self.vault_url, - VAULT_TOKEN=self.vault_token, + TF_VAR_vault_address=self.vault_url, + TF_VAR_vault_token=self.vault_token, ) self.terraform_backend == TERRAFORM_BACKEND_TFC and env.update( TF_VAR_terraform_cloud_token=self.terraform_cloud_token @@ -733,13 +750,9 @@ def run_terraform(self, module_name, env, outputs=None): def make_sed(self, file_path, placeholder, replace_value): """Replace a placeholder value with a given one in a given file.""" - subprocess.run( - [ - "sed", - "-i", - f"s/{placeholder}/{replace_value}/", - str(self.output_dir / self.project_dirname / file_path), - ] + target_file = self.output_dir / self.project_dirname / file_path + target_file.write_text( + target_file.read_text().replace(placeholder, replace_value) ) def init_subrepo(self, service_slug, template_url, **kwargs): @@ -791,6 +804,7 @@ def init_subrepo(self, service_slug, template_url, **kwargs): } subprocess.run( ["python", "-m", "pip", "install", "-r", "requirements/common.txt"], + capture_output=True, cwd=subrepo_dir, ) subprocess.run( @@ -825,10 +839,10 @@ def run(self): click.echo(highlight(f"Initializing the {self.service_slug} service:")) self.init_service() self.create_env_file() - if self.gitlab_group_slug: - self.init_gitlab() if self.terraform_backend == TERRAFORM_BACKEND_TFC: self.init_terraform_cloud() + if self.gitlab_group_slug: + self.init_gitlab() if self.vault_url: self.init_vault() frontend_template_url = FRONTEND_TEMPLATE_URLS.get(self.frontend_type) diff --git a/cookiecutter.json b/cookiecutter.json index 632214c4..ac36c0a4 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -3,32 +3,62 @@ "project_slug": "{{ cookiecutter.project_name | slugify() }}", "project_dirname": "{{ cookiecutter.project_slug | slugify(separator='') }}", "service_slug": "orchestrator", - "backend_type": ["django", "none"], + "backend_type": [ + "django", + "none" + ], "backend_service_slug": "backend", "backend_service_port": "8000", - "frontend_type": ["nextjs", "none"], + "frontend_type": [ + "nextjs", + "none" + ], "frontend_service_slug": "frontend", "frontend_service_port": "3000", "terraform_backend": "gitlab", "terraform_cloud_organization": "", - "media_storage": ["digitalocean-s3", "aws-s3", "local", "none"], + "media_storage": [ + "digitalocean-s3", + "aws-s3", + "local", + "none" + ], "use_pact": "false", "use_vault": "false", - "stacks": { - "main": { - "dev": { - "name": "development" + "resources": { + "stacks": [ + [ + { + "name": "main", + "slug": "main" + } + ] + ], + "envs": [ + { + "name": "development", + "slug": "dev", + "stack_slug": "main" }, - "stage": { - "name": "staging" + { + "name": "staging", + "slug": "stage", + "stack_slug": "main" }, - "prod": { - "name": "production" + { + "name": "production", + "slug": "prod", + "stack_slug": "main" } - } + ] }, - "deployment_type": ["digitalocean-k8s", "other-k8s"], + "deployment_type": [ + "digitalocean-k8s", + "other-k8s" + ], "environment_distribution": "1", "tfvars": {}, - "_extensions": ["cookiecutter.extensions.SlugifyExtension"] -} + "_extensions": [ + "cookiecutter.extensions.SlugifyExtension" + ] +} \ No newline at end of file diff --git a/start.py b/start.py index d0fcc471..18093cde 100755 --- a/start.py +++ b/start.py @@ -12,6 +12,7 @@ ENVIRONMENT_DISTRIBUTION_CHOICES, GITLAB_TOKEN_ENV_VAR, MEDIA_STORAGE_CHOICES, + VAULT_TOKEN_ENV_VAR, ) from bootstrap.exceptions import BootstrapError from bootstrap.helpers import slugify_option @@ -53,7 +54,7 @@ default=None, ) @click.option("--terraform-cloud-admin-email") -@click.option("--vault-token") +@click.option("--vault-token", envvar=VAULT_TOKEN_ENV_VAR) @click.option("--vault-url") @click.option("--digitalocean-token") @click.option( diff --git a/terraform/gitlab/main.tf b/terraform/gitlab/main.tf index bb93320b..41c3f060 100644 --- a/terraform/gitlab/main.tf +++ b/terraform/gitlab/main.tf @@ -6,27 +6,20 @@ locals { git_config_args = "-c user.email=${local.user_data.email} -c user.name=\"${local.user_data.name}\"" + escaped_base_ssh_url = replace(replace(gitlab_project.main.ssh_url_to_repo, "/${var.project_slug}.git", ""), "/", "\\/") + reserved_member_ids = toset([tostring(local.user_data.id)]) owners = setsubtract( toset([for i in data.gitlab_users.owners : tostring(i.users[0].id) if length(i.users) > 0]), local.reserved_member_ids, ) maintainers = setsubtract( - setsubtract( - toset([for i in data.gitlab_users.maintainers : tostring(i.users[0].id) if length(i.users) > 0]), - local.owners - ), - local.reserved_member_ids, + toset([for i in data.gitlab_users.maintainers : tostring(i.users[0].id) if length(i.users) > 0]), + setunion(local.reserved_member_ids, local.owners), ) developers = setsubtract( - setsubtract( - setsubtract( - toset([for i in data.gitlab_users.developers : tostring(i.users[0].id) if length(i.users) > 0]), - local.maintainers - ), - local.owners - ), - local.reserved_member_ids, + toset([for i in data.gitlab_users.developers : tostring(i.users[0].id) if length(i.users) > 0]), + setunion(local.reserved_member_ids, local.owners, local.maintainers), ) } @@ -103,6 +96,7 @@ resource "null_resource" "init_repo" { "cd ${var.local_repository_dir}", format( join(" && ", [ + "sed -i 's/__VCS_BASE_SSH_URL__/${local.escaped_base_ssh_url}/' README.md", "git init --initial-branch=main", "git remote add origin %s", "git add .", diff --git a/terraform/gitlab/outputs.tf b/terraform/gitlab/outputs.tf index 9c6e5d26..3c46c636 100644 --- a/terraform/gitlab/outputs.tf +++ b/terraform/gitlab/outputs.tf @@ -9,8 +9,3 @@ output "registry_username" { sensitive = true value = gitlab_deploy_token.regcred.username } - -output "ssh_url_to_repo" { - description = "The SSH URL to the project repository." - value = gitlab_project.main.ssh_url_to_repo -} diff --git a/terraform/terraform-cloud/variables.tf b/terraform/terraform-cloud/variables.tf index 0b433609..193c7e4a 100644 --- a/terraform/terraform-cloud/variables.tf +++ b/terraform/terraform-cloud/variables.tf @@ -13,6 +13,7 @@ variable "create_organization" { variable "environments" { description = "The list of environments slugs." type = list(string) + default = [] } variable "hostname" { @@ -44,6 +45,7 @@ variable "service_slug" { variable "stacks" { description = "The list of stacks slugs." type = list(string) + default = [] } variable "terraform_cloud_token" { diff --git a/terraform/vault-admin/main.tf b/terraform/vault-admin/main.tf deleted file mode 100644 index 8afcf9e9..00000000 --- a/terraform/vault-admin/main.tf +++ /dev/null @@ -1,197 +0,0 @@ -locals { - auth_slug = "gitlab-jwt-${var.project_slug}" -} - - -terraform { - required_providers { - vault = { - source = "hashicorp/vault" - version = "3.7.0" - } - } -} - -provider "vault" { -} - -/* GitLab JWT Auth */ - -resource "vault_jwt_auth_backend" "gitlab_jwt" { - description = "Demonstration of the Terraform JWT auth backend" - path = local.auth_slug - jwks_url = format("%s/-/jwks", trimsuffix(var.gitlab_url, "/")) - bound_issuer = var.gitlab_url -} - -resource "vault_policy" "gitlab_jwt_stacks" { - for_each = var.stacks_environments - - name = "${local.auth_slug}-stacks-${each.key}" - - policy = < keys(v) }) - - name = "${local.auth_slug}-envs-${each.key}" - - policy = < /dev/null || echo {}) > ${secret_var_file} - var_files="${var_files} -var-file=${secret_var_file}" - done -} - -if [ "${STACK_SLUG}" != "" ] && [ "${VAULT_STACK_SECRETS}" != "" ]; then - load_secrets "stacks" ${STACK_SLUG} "${VAULT_STACK_SECRETS}" -fi - -if [ "${ENV_SLUG}" != "" ] && [ "${VAULT_ENV_SECRETS}" != "" ]; then - load_secrets "envs" ${ENV_SLUG} "${VAULT_ENV_SECRETS}" -fi +for secret_path in ${VAULT_SECRETS} +do + secret_var_file=${TERRAFORM_VARS_DIR}/`echo ${secret_path} | tr / -`.json + curl --silent --header "X-Vault-Token: ${VAULT_TOKEN}" ${VAULT_ADDR%/}/v1/${PROJECT_SLUG}/${VAULT_SECRETS_PREFIX}/${secret_path} | jq -r ".data // {}" > ${secret_var_file} + var_files="${var_files} -var-file=${secret_var_file}" +done if [ "${TERRAFORM_BACKEND}" == "terraform-cloud" ]; then - export TFC_TOKEN="$(./vault read -field=token ${PROJECT_SLUG}-tfc/creds/default)" + export TFC_TOKEN=`curl --silent --header "X-Vault-Token: ${VAULT_TOKEN}" ${VAULT_ADDR%/}/v1/${PROJECT_SLUG}-tfc/creds/default | jq -r .data.token ` fi