diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da80e4d8..d8240a32 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,12 +26,12 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: "v2.31.1" + rev: "v2.34.0" hooks: - id: pyupgrade args: [--py310-plus] - repo: https://github.com/psf/black - rev: "22.1.0" + rev: "22.3.0" hooks: - id: black - repo: https://github.com/pycqa/isort @@ -48,6 +48,6 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-bugbear~=22.1.11 + - flake8-bugbear~=22.4.25 - flake8-docstrings~=1.6.0 - flake8-isort~=4.1.0 diff --git a/Makefile b/Makefile index f047985a..97d9d051 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ pip: pip_update ## Compile requirements .PHONY: pip_update pip_update: ## Update requirements and dependencies - python3 -m pip install -q -U pip~=22.0.0 pip-tools~=6.5.0 setuptools~=60.10.0 wheel~=0.37.0 + python3 -m pip install -q -U pip~=22.1.0 pip-tools~=6.6.0 setuptools~=60.10.0 wheel~=0.37.0 .PHONY: precommit precommit: ## Fix code formatting, linting and sorting imports diff --git a/README.md b/README.md index c9c0caab..75f21e86 100644 --- a/README.md +++ b/README.md @@ -195,10 +195,10 @@ The following arguments can be appended to the Docker and shell commands #### Frontend type -| Value | Description | Argument | -| ------ | ----------------------------------------------------- | ------------------------ | +| Value | Description | Argument | +| ------ | --------------------------------------------------- | ------------------------ | | nextjs | https://github.com/20tab/nextjs-continuous-delivery | `--frontend-type=nextjs` | -| none | the frontend service will not be initialized | `--frontend-type=none` | +| none | the frontend service will not be initialized | `--frontend-type=none` | #### Frontend service slug diff --git a/bootstrap/collector.py b/bootstrap/collector.py index b8c90754..fc3a5628 100755 --- a/bootstrap/collector.py +++ b/bootstrap/collector.py @@ -26,6 +26,7 @@ ENVIRONMENT_DISTRIBUTION_PROMPT, FRONTEND_TYPE_CHOICES, FRONTEND_TYPE_DEFAULT, + GITLAB_URL_DEFAULT, MEDIA_STORAGE_AWS_S3, MEDIA_STORAGE_CHOICES, MEDIA_STORAGE_DIGITALOCEAN_S3, @@ -58,6 +59,8 @@ def collect( terraform_cloud_organization, terraform_cloud_organization_create, terraform_cloud_admin_email, + vault_token, + vault_url, digitalocean_token, kubernetes_cluster_ca_certificate, kubernetes_host, @@ -99,6 +102,7 @@ def collect( s3_access_id, s3_secret_key, s3_bucket_name, + gitlab_url, gitlab_private_token, gitlab_group_slug, gitlab_group_owners, @@ -144,6 +148,7 @@ def collect( terraform_cloud_organization_create, terraform_cloud_admin_email, ) + vault_token, vault_url = clean_vault_data(vault_token, vault_url, quiet) environment_distribution = clean_environment_distribution( environment_distribution, deployment_type ) @@ -242,6 +247,7 @@ def collect( pact_broker_url, pact_broker_username, pact_broker_password ) ( + gitlab_url, gitlab_group_slug, gitlab_private_token, gitlab_group_owners, @@ -249,6 +255,7 @@ def collect( gitlab_group_developers, ) = clean_gitlab_group_data( project_slug, + gitlab_url, gitlab_group_slug, gitlab_private_token, gitlab_group_owners, @@ -256,8 +263,7 @@ def collect( gitlab_group_developers, quiet, ) - # TODO: change when moving secrets to Vault - if gitlab_group_slug and "s3" in media_storage: + if (gitlab_url or vault_url) and "s3" in media_storage: ( digitalocean_token, s3_region, @@ -295,6 +301,8 @@ def collect( "terraform_cloud_organization": terraform_cloud_organization, "terraform_cloud_organization_create": terraform_cloud_organization_create, "terraform_cloud_admin_email": terraform_cloud_admin_email, + "vault_token": vault_token, + "vault_url": vault_url, "digitalocean_token": digitalocean_token, "kubernetes_cluster_ca_certificate": kubernetes_cluster_ca_certificate, "kubernetes_host": kubernetes_host, @@ -340,6 +348,7 @@ def collect( "s3_access_id": s3_access_id, "s3_secret_key": s3_secret_key, "s3_bucket_name": s3_bucket_name, + "gitlab_url": gitlab_url, "gitlab_private_token": gitlab_private_token, "gitlab_group_slug": gitlab_group_slug, "gitlab_group_owners": gitlab_group_owners, @@ -491,7 +500,7 @@ def clean_terraform_backend( terraform_cloud_organization_create, terraform_cloud_admin_email, ): - """Return the terraform backend and the Terraform Cloud data, if applicable.""" + """Return the Terraform backend and the Terraform Cloud data, if applicable.""" terraform_backend = ( terraform_backend if terraform_backend in TERRAFORM_BACKEND_CHOICES @@ -545,6 +554,29 @@ def clean_terraform_backend( ) +def clean_vault_data(vault_token, vault_url, quiet=False): + """Return the Vault data, if applicable.""" + if vault_token or ( + vault_token is None + and click.confirm( + "Do you want to use Vault for secrets management?", + ) + ): + vault_token = validate_or_prompt_password("Vault token", vault_token) + quiet or click.confirm( + warning( + "Make sure the Vault token has enough permissions to enable the " + "project secrets backends and manage the project secrets. Continue?" + ), + abort=True, + ) + vault_url = validate_or_prompt_url("Vault address", vault_url) + else: + vault_token = None + vault_url = None + return vault_token, vault_url + + def clean_environment_distribution(environment_distribution, deployment_type): """Return the environment distribution.""" if deployment_type == DEPLOYMENT_TYPE_OTHER: @@ -864,6 +896,7 @@ def clean_media_storage(media_storage): def clean_gitlab_group_data( project_slug, + gitlab_url, gitlab_group_slug, gitlab_private_token, gitlab_group_owners, @@ -872,10 +905,13 @@ def clean_gitlab_group_data( quiet=False, ): """Return GitLab group data.""" - if gitlab_group_slug or ( - gitlab_group_slug is None + if gitlab_url or ( + gitlab_url is None and click.confirm(warning("Do you want to use GitLab?"), default=True) ): + gitlab_url = validate_or_prompt_url( + "GitLab URL", gitlab_url, default=GITLAB_URL_DEFAULT + ) gitlab_group_slug = slugify( gitlab_group_slug or click.prompt("GitLab group slug", default=project_slug) ) @@ -905,12 +941,14 @@ def clean_gitlab_group_data( else click.prompt("Comma-separated GitLab group developers", default="") ) else: + gitlab_url = None gitlab_group_slug = None gitlab_private_token = None gitlab_group_owners = None gitlab_group_maintainers = None gitlab_group_developers = None return ( + gitlab_url, gitlab_group_slug, gitlab_private_token, gitlab_group_owners, diff --git a/bootstrap/constants.py b/bootstrap/constants.py index f3118c15..4ce88803 100755 --- a/bootstrap/constants.py +++ b/bootstrap/constants.py @@ -4,6 +4,28 @@ DUMPS_DIR = Path(__file__).parent.parent / ".dumps" +# Stacks + +DEV_STACK_SLUG = "dev" + +STAGE_STACK_SLUG = "stage" + +MAIN_STACK_SLUG = "main" + +# Environments + +DEV_ENV_NAME = "development" + +DEV_ENV_SLUG = "dev" + +STAGE_ENV_NAME = "staging" + +STAGE_ENV_SLUG = "stage" + +PROD_ENV_NAME = "production" + +PROD_ENV_SLUG = "prod" + # Env vars GITLAB_TOKEN_ENV_VAR = "GITLAB_PRIVATE_TOKEN" @@ -18,11 +40,11 @@ "nextjs": "https://github.com/20tab/nextjs-continuous-delivery" } -SUBREPOS_DIR = ".subrepos" +SUBREPOS_DIR = Path(__file__).parent.parent / ".subrepos" # Services type -ORCHESTRATOR_SERVICE_SLUG = "orchestrator" +SERVICE_SLUG_DEFAULT = "orchestrator" EMPTY_SERVICE_TYPE = "none" @@ -86,3 +108,7 @@ TERRAFORM_BACKEND_TFC = "terraform-cloud" TERRAFORM_BACKEND_CHOICES = [TERRAFORM_BACKEND_TFC, TERRAFORM_BACKEND_GITLAB] + +# GitLab + +GITLAB_URL_DEFAULT = "https://gitlab.com" diff --git a/bootstrap/helpers.py b/bootstrap/helpers.py index d14c0043..2437559a 100644 --- a/bootstrap/helpers.py +++ b/bootstrap/helpers.py @@ -3,6 +3,16 @@ from slugify import slugify +def format_gitlab_variable(value, masked=False, protected=True): + """Format the given value to be used as a Terraform variable.""" + return ( + f'{{ value = "{value}"' + + (masked and ", masked = true" or "") + + (not protected and ", protected = false" or "") + + "}" + ) + + def format_tfvar(value, value_type=None): """Format the given value to be used as a Terraform variable.""" if value_type == "list": diff --git a/bootstrap/runner.py b/bootstrap/runner.py index d919fedd..7f176b3a 100644 --- a/bootstrap/runner.py +++ b/bootstrap/runner.py @@ -19,16 +19,25 @@ from bootstrap.constants import ( BACKEND_TEMPLATE_URLS, DEPLOYMENT_TYPE_OTHER, + DEV_ENV_NAME, + DEV_ENV_SLUG, + DEV_STACK_SLUG, DUMPS_DIR, FRONTEND_TEMPLATE_URLS, - MEDIA_STORAGE_AWS_S3, + GITLAB_URL_DEFAULT, + MAIN_STACK_SLUG, MEDIA_STORAGE_DIGITALOCEAN_S3, - ORCHESTRATOR_SERVICE_SLUG, + PROD_ENV_NAME, + PROD_ENV_SLUG, + SERVICE_SLUG_DEFAULT, + STAGE_ENV_NAME, + STAGE_ENV_SLUG, + STAGE_STACK_SLUG, SUBREPOS_DIR, TERRAFORM_BACKEND_TFC, ) from bootstrap.exceptions import BootstrapError -from bootstrap.helpers import format_tfvar +from bootstrap.helpers import format_gitlab_variable, format_tfvar error = partial(click.style, fg="red") @@ -62,6 +71,8 @@ class Runner: terraform_cloud_organization: str | None = None terraform_cloud_organization_create: bool | None = None terraform_cloud_admin_email: str | None = None + vault_token: str | None = None + vault_url: str | None = None digitalocean_token: str | None = None kubernetes_cluster_ca_certificate: str | None = None kubernetes_host: str | None = None @@ -103,6 +114,7 @@ class Runner: s3_access_id: str | None = None s3_secret_key: str | None = None s3_bucket_name: str | None = None + gitlab_url: str | None = None gitlab_private_token: str | None = None gitlab_group_slug: str | None = None gitlab_group_owners: str | None = None @@ -115,82 +127,205 @@ class Runner: run_id: str = field(init=False) service_slug: str = field(init=False) stacks_environments: dict = field(init=False, default_factory=dict) + gitlab_variables: dict = field(init=False, default_factory=dict) tfvars: dict = field(init=False, default_factory=dict) + vault_project_path: str = field(init=False, default="") + vault_secrets: dict = field(init=False, default_factory=dict) + terraform_run_modules: list = field(init=False, default_factory=list) + terraform_outputs: dict = field(init=False, default_factory=dict) def __post_init__(self): """Finalize initialization.""" - self.service_slug = ORCHESTRATOR_SERVICE_SLUG + self.service_slug = SERVICE_SLUG_DEFAULT + self.gitlab_url = self.gitlab_url and self.gitlab_url.rstrip("/") self.run_id = f"{time():.0f}" self.terraform_dir = self.terraform_dir or Path(f".terraform/{self.run_id}") self.logs_dir = self.logs_dir or Path(f".logs/{self.run_id}") + self.set_vault_project_path() self.set_stacks_environments() - self.set_tfvars() + self.collect_tfvars() + self.collect_gitlab_variables() + + def set_vault_project_path(self): + """Set the Vault project path.""" + if self.vault_url: + self.vault_project_path = self.gitlab_group_slug or self.project_slug def set_stacks_environments(self): """Set the environments distribution per stack.""" dev_env = { - "name": "Development", - "url": self.project_url_dev, + "name": DEV_ENV_NAME, "prefix": self.subdomain_dev, + "url": self.project_url_dev, } stage_env = { - "name": "Staging", - "url": self.project_url_stage, + "name": STAGE_ENV_NAME, "prefix": self.subdomain_stage, + "url": self.project_url_stage, } prod_env = { - "name": "Production", - "url": self.project_url_prod, + "name": PROD_ENV_NAME, "prefix": self.subdomain_prod, + "url": self.project_url_prod, } if self.environment_distribution == "1": self.stacks_environments = { - "main": {"dev": dev_env, "stage": stage_env, "prod": prod_env} + MAIN_STACK_SLUG: { + DEV_ENV_SLUG: dev_env, + STAGE_ENV_SLUG: stage_env, + PROD_ENV_SLUG: prod_env, + } } elif self.environment_distribution == "2": self.stacks_environments = { - "dev": {"dev": dev_env, "stage": stage_env}, - "main": {"prod": prod_env}, + DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env, STAGE_ENV_SLUG: stage_env}, + MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env}, } elif self.environment_distribution == "3": self.stacks_environments = { - "dev": {"dev": dev_env}, - "stage": {"stage": stage_env}, - "main": {"prod": prod_env}, + DEV_STACK_SLUG: {DEV_ENV_SLUG: dev_env}, + STAGE_STACK_SLUG: {STAGE_ENV_SLUG: stage_env}, + MAIN_STACK_SLUG: {PROD_ENV_SLUG: prod_env}, } - def add_tfvar(self, tf_stage, var_name, var_value=None, var_type=None): - """Add a Terraform value to the given .tfvars file.""" + def register_gitlab_variable( + self, level, var_name, var_value=None, masked=False, protected=True + ): + """Register a GitLab variable at the given level.""" + vars_dict = self.gitlab_variables.setdefault(level, {}) + if var_value is None: + var_value = getattr(self, var_name) + vars_dict[var_name] = format_gitlab_variable(var_value, masked, protected) + + def register_gitlab_variables(self, level, *vars): + """Register one or more GitLab variable at the given level.""" + [ + self.register_gitlab_variable(level, *((i,) if isinstance(i, str) else i)) + for i in vars + ] + + def register_gitlab_group_variables(self, *vars): + """Register one or more GitLab group variable.""" + self.register_gitlab_variables("group", *vars) + + def register_gitlab_project_variables(self, *vars): + """Register one or more GitLab project variable.""" + self.register_gitlab_variables("project", *vars) + + def collect_gitlab_variables(self): + """Collect the GitLab group and project variables.""" + if self.pact_broker_url: + self.register_gitlab_group_variables(("PACT_ENABLED", "true", False, False)) + if self.vault_token: + self.register_gitlab_group_variables( + ("VAULT_ADDR", self.vault_url, False, False) + ) + else: + self.collect_gitlab_variables_secrets() + + def collect_gitlab_variables_secrets(self): + """Collect secrets as GitLab group and project variables.""" + self.register_gitlab_group_variables( + ("BASIC_AUTH_USERNAME", self.project_slug), + ("BASIC_AUTH_PASSWORD", secrets.token_urlsafe(12), True), + ) + self.sentry_org and self.register_gitlab_group_variables( + ("SENTRY_AUTH_TOKEN", self.sentry_auth_token, True) + ) + if self.pact_broker_url: + pact_broker_url = self.pact_broker_url + pact_broker_username = self.pact_broker_username + pact_broker_password = self.pact_broker_password + pact_broker_auth_url = re.sub( + r"^(https?)://(.*)$", + rf"\g<1>://{pact_broker_username}:{pact_broker_password}@\g<2>", + pact_broker_url, + ) + self.register_gitlab_group_variables( + ("PACT_BROKER_BASE_URL", pact_broker_url, False, False), + ("PACT_BROKER_USERNAME", pact_broker_username, False, False), + ("PACT_BROKER_PASSWORD", pact_broker_password, True, False), + ("PACT_BROKER_AUTH_URL", pact_broker_auth_url, True, False), + ) + if self.terraform_backend == TERRAFORM_BACKEND_TFC: + self.register_gitlab_group_variables( + ("TFC_TOKEN", self.terraform_cloud_token, True) + ) + if self.subdomain_monitoring: + self.register_gitlab_project_variables( + ("GRAFANA_PASSWORD", secrets.token_urlsafe(12), True) + ) + self.digitalocean_token and self.register_gitlab_group_variables( + ("DIGITALOCEAN_TOKEN", self.digitalocean_token, True) + ) + if self.deployment_type == DEPLOYMENT_TYPE_OTHER: + self.register_gitlab_group_variables( + ( + "KUBERNETES_CLUSTER_CA_CERTIFICATE", + base64.b64encode( + Path(self.kubernetes_cluster_ca_certificate).read_bytes() + ).decode(), + True, + ), + ("KUBERNETES_HOST", self.kubernetes_host), + ("KUBERNETES_TOKEN", self.kubernetes_token, True), + ) + if "s3" in self.media_storage: + self.register_gitlab_group_variables( + ("S3_ACCESS_ID", self.s3_access_id, True), + ("S3_REGION", self.s3_region), + ("S3_SECRET_KEY", self.s3_secret_key, True), + ) + self.s3_bucket_name and self.register_gitlab_group_variables( + ("S3_BUCKET_NAME", self.s3_bucket_name) + ) + ( + self.media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3 + and self.register_gitlab_group_variables(("S3_HOST", self.s3_host)) + ) + + def render_gitlab_variables_to_string(self, level): + """Return the given level GitLab variables rendered to string.""" + return "{%s}" % ", ".join( + f"{k} = {v}" for k, v in self.gitlab_variables.get(level, {}).items() + ) + + def register_tfvar(self, tf_stage, var_name, var_value=None, var_type=None): + """Register a Terraform variable value for the given stage.""" vars_list = self.tfvars.setdefault(tf_stage, []) if var_value is None: var_value = getattr(self, var_name) vars_list.append("=".join((var_name, format_tfvar(var_value, var_type)))) - def add_tfvars(self, tf_stage, *vars): - """Add one or more Terraform variables to the given stage.""" - [self.add_tfvar(tf_stage, *((i,) if isinstance(i, str) else i)) for i in vars] + def register_tfvars(self, tf_stage, *vars): + """Register one or more Terraform variable for the given stage.""" + [ + self.register_tfvar(tf_stage, *((i,) if isinstance(i, str) else i)) + for i in vars + ] - def add_base_tfvars(self, *vars, stack_slug=None): - """Add one or more base Terraform variables.""" + def register_base_tfvars(self, *vars, stack_slug=None): + """Register one or more base Terraform variable.""" tf_stage = "base" + (stack_slug and f"_{stack_slug}" or "") - self.add_tfvars(tf_stage, *vars) + self.register_tfvars(tf_stage, *vars) - def add_cluster_tfvars(self, *vars, stack_slug=None): - """Add one or more cluster Terraform variables.""" + def register_cluster_tfvars(self, *vars, stack_slug=None): + """Register one or more cluster Terraform variable.""" tf_stage = "cluster" + (stack_slug and f"_{stack_slug}" or "") - self.add_tfvars(tf_stage, *vars) + self.register_tfvars(tf_stage, *vars) - def add_environment_tfvars(self, *vars, env_slug=None): - """Add one or more environment Terraform variables.""" + def register_environment_tfvars(self, *vars, env_slug=None): + """Register one or more environment Terraform variable.""" tf_stage = "environment" + (env_slug and f"_{env_slug}" or "") - self.add_tfvars(tf_stage, *vars) + self.register_tfvars(tf_stage, *vars) - def set_tfvars(self): - """Set base, cluster and environment Terraform variables lists.""" + def collect_tfvars(self): + """Collect the base, cluster and environment Terraform variables.""" + self.register_environment_tfvars(("registry_server", "registry.gitlab.com")) backend_service_paths = ["/"] frontend_service_paths = ["/"] if self.frontend_service_slug: - self.add_environment_tfvars( + self.register_environment_tfvars( ("frontend_service_paths", frontend_service_paths, "list"), ("frontend_service_port", None, "num"), "frontend_service_slug", @@ -199,29 +334,29 @@ def set_tfvars(self): ["/media"] if self.media_storage == "local" else [] ) if self.backend_service_slug: - self.add_environment_tfvars( + self.register_environment_tfvars( ("backend_service_paths", backend_service_paths, "list"), ("backend_service_port", None, "num"), "backend_service_slug", ) - self.project_domain and self.add_environment_tfvars("project_domain") + self.project_domain and self.register_environment_tfvars("project_domain") if self.letsencrypt_certificate_email: - self.add_cluster_tfvars("letsencrypt_certificate_email") - self.add_environment_tfvars("letsencrypt_certificate_email") - self.subdomain_monitoring and self.add_environment_tfvars( + self.register_cluster_tfvars("letsencrypt_certificate_email") + self.register_environment_tfvars("letsencrypt_certificate_email") + self.subdomain_monitoring and self.register_environment_tfvars( ("monitoring_subdomain", self.subdomain_monitoring), env_slug="prod" ) if self.use_redis: - self.add_base_tfvars(("use_redis", True, "bool")) - self.add_environment_tfvars(("use_redis", True, "bool")) + self.register_base_tfvars(("use_redis", True, "bool")) + self.register_environment_tfvars(("use_redis", True, "bool")) if "digitalocean" in self.deployment_type: - self.add_environment_tfvars( + self.register_environment_tfvars( ("create_dns_records", self.digitalocean_dns_records_create, "bool"), ) - self.digitalocean_domain_create and self.add_environment_tfvars( - ("create_domain", True, "bool"), env_slug="dev" + self.digitalocean_domain_create and self.register_environment_tfvars( + ("create_domain", True, "bool"), env_slug=DEV_ENV_SLUG ) - self.add_base_tfvars( + self.register_base_tfvars( ("k8s_cluster_region", self.digitalocean_k8s_cluster_region), ("database_cluster_region", self.digitalocean_database_cluster_region), ( @@ -229,32 +364,130 @@ def set_tfvars(self): self.digitalocean_database_cluster_node_size, ), ) - self.use_redis and self.add_base_tfvars( + self.use_redis and self.register_base_tfvars( ("redis_cluster_region", self.digitalocean_redis_cluster_region), ("redis_cluster_node_size", self.digitalocean_redis_cluster_node_size), ) elif self.deployment_type == DEPLOYMENT_TYPE_OTHER: - self.add_environment_tfvars( + self.register_environment_tfvars( "postgres_image", "postgres_persistent_volume_capacity", "postgres_persistent_volume_claim_capacity", "postgres_persistent_volume_host_path", ) - self.use_redis and self.add_environment_tfvars("redis_image") + self.use_redis and self.register_environment_tfvars("redis_image") if self.media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3: - self.add_base_tfvars(("create_s3_bucket", True, "bool")) - self.add_environment_tfvars( + self.register_base_tfvars(("create_s3_bucket", True, "bool")) + self.register_environment_tfvars( ("digitalocean_spaces_bucket_available", True, "bool") ) for stack_slug, stack_envs in self.stacks_environments.items(): - for env_slug, env_data in stack_envs.items(): - self.add_environment_tfvars( + for env_slug, _env_data in stack_envs.items(): + self.register_environment_tfvars( ("basic_auth_enabled", env_slug != "prod", "bool"), ("stack_slug", stack_slug), ("subdomains", [getattr(self, f"subdomain_{env_slug}")], "list"), env_slug=env_slug, ) + def register_vault_stack_secret(self, stack_slug, name, data): + """Register a Vault stack secret locally.""" + self.vault_secrets[f"stacks/{stack_slug}/{name}"] = data + + def register_vault_environment_secret(self, env_slug, name, data): + """Register a Vault environment secret locally.""" + self.vault_secrets[f"envs/{env_slug}/{name}"] = data + + def collect_vault_stack_secrets(self, stack_slug): + """Collect the Vault secrets for the given stack.""" + self.digitalocean_token and self.register_vault_stack_secret( + stack_slug, "digitalocean", dict(digitalocean_token=self.digitalocean_token) + ) + if "s3" in self.media_storage: + s3_secret = dict( + s3_region=self.s3_region, + s3_access_id=self.s3_access_id, + s3_secret_key=self.s3_secret_key, + ) + self.s3_bucket_name and s3_secret.update(s3_bucket_name=self.s3_bucket_name) + self.media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3 and s3_secret.update( + s3_host=self.s3_host + ) + self.register_vault_stack_secret(stack_slug, "s3", s3_secret) + ( + self.subdomain_monitoring + and stack_slug == MAIN_STACK_SLUG + and self.register_vault_stack_secret( + stack_slug, + "monitoring", + dict(grafana_password=secrets.token_urlsafe(12)), + ) + ) + ( + self.deployment_type == DEPLOYMENT_TYPE_OTHER + and self.register_vault_stack_secret( + stack_slug, + "k8s", + dict( + kubernetes_cluster_ca_certificate=base64.b64encode( + Path(self.kubernetes_cluster_ca_certificate).read_bytes() + ).decode(), + kubernetes_host=self.kubernetes_host, + kubernetes_token=self.kubernetes_token, + ), + ) + ) + + def collect_vault_environment_secrets(self, env_slug): + """Collect the Vault secrets for the given environment.""" + self.register_vault_environment_secret( + env_slug, + "basic_auth", + dict( + basic_auth_username=self.project_slug, + basic_auth_password=secrets.token_urlsafe(12), + ), + ) + # Sentry secrets are used by the GitLab CI/CD + self.sentry_org and self.register_vault_environment_secret( + env_slug, "sentry", dict(sentry_auth_token=self.sentry_auth_token) + ) + + def collect_vault_pact_secrets(self): + """Collect the Vault secrets for Pact.""" + # Pact secrets are used by the GitLab CI/CD + pact_broker_url = self.pact_broker_url + pact_broker_username = self.pact_broker_username + pact_broker_password = self.pact_broker_password + pact_broker_auth_url = re.sub( + r"^(https?)://(.*)$", + rf"\g<1>://{pact_broker_username}:{pact_broker_password}@\g<2>", + pact_broker_url, + ) + self.vault_secrets["pact"] = dict( + pact_broker_base_url=pact_broker_url, + pact_broker_username=pact_broker_username, + pact_broker_password=pact_broker_password, + pact_broker_auth_url=pact_broker_auth_url, + ) + + def collect_vault_secrets(self): + """Collect Vault secrets.""" + regcred = None + if gitlab_terraform_outputs := self.terraform_outputs.get("gitlab"): + regcred = dict( + registry_username=gitlab_terraform_outputs["registry_username"], + registry_password=gitlab_terraform_outputs["registry_password"], + ) + for stack_slug, stack_envs in self.stacks_environments.items(): + self.collect_vault_stack_secrets(stack_slug) + for env_slug in stack_envs: + self.collect_vault_environment_secrets(env_slug) + regcred and self.register_vault_environment_secret( + env_slug, "regcred", regcred + ) + self.pact_broker_url and self.collect_vault_pact_secrets() + def init_service(self): """Initialize the service.""" click.echo(info("...cookiecutting the service")) @@ -272,13 +505,14 @@ def init_service(self): "media_storage": self.media_storage, "pact_enabled": bool(self.pact_broker_url), "project_dirname": self.project_dirname, - "project_domain": self.project_domain, "project_name": self.project_name, "project_slug": self.project_slug, + "service_slug": self.service_slug, "stacks": self.stacks_environments, "terraform_backend": self.terraform_backend, "terraform_cloud_organization": self.terraform_cloud_organization, "tfvars": self.tfvars, + "vault_project_path": self.vault_project_path, }, output_dir=self.output_dir, no_input=True, @@ -295,9 +529,206 @@ def create_env_file(self): ) (self.service_dir / ".env").write_text(env_text) + def init_gitlab(self): + """Initialize the GitLab resources.""" + click.echo(info("...creating the GitLab resources with Terraform")) + env = dict( + TF_VAR_gitlab_token=self.gitlab_private_token, + TF_VAR_group_maintainers=self.gitlab_group_maintainers, + TF_VAR_group_name=self.project_name, + TF_VAR_group_owners=self.gitlab_group_owners, + TF_VAR_group_slug=self.gitlab_group_slug, + TF_VAR_group_variables=self.render_gitlab_variables_to_string("group"), + TF_VAR_local_repository_dir=self.service_dir, + TF_VAR_project_description=( + f'The "{self.project_name}" project {self.service_slug} service.' + ), + TF_VAR_project_name=self.service_slug.title(), + TF_VAR_project_slug=self.service_slug, + TF_VAR_project_variables=self.render_gitlab_variables_to_string("project"), + TF_VAR_vault_enabled=self.vault_url and "true" or "false", + ) + self.gitlab_url != GITLAB_URL_DEFAULT and env.update( + GITLAB_BASE_URL=f"{self.gitlab_url}/api/v4/" + ) + self.run_terraform( + "gitlab", env, outputs=["registry_password", "registry_username"] + ) + + def init_terraform_cloud(self): + """Initialize the Terraform Cloud resources.""" + click.echo(info("...creating the Terraform Cloud resources with Terraform")) + env = dict( + TF_VAR_admin_email=self.terraform_cloud_admin_email, + TF_VAR_create_organization=self.terraform_cloud_organization_create + and "true" + or "false", + TF_VAR_environments=json.dumps( + [i for j in self.stacks_environments.values() for i in j] + ), + TF_VAR_hostname=self.terraform_cloud_hostname, + TF_VAR_organization_name=self.terraform_cloud_organization, + TF_VAR_project_name=self.project_name, + TF_VAR_project_slug=self.project_slug, + TF_VAR_service_slug=self.service_slug, + TF_VAR_stacks=json.dumps(list(self.stacks_environments)), + TF_VAR_terraform_cloud_token=self.terraform_cloud_token, + ) + self.run_terraform("terraform-cloud", env) + + def init_vault(self): + """Initialize the Vault resources.""" + click.echo(info("...creating the Vault resources with Terraform")) + # NOTE: Vault secrets collection must be done AFTER GitLab init + self.collect_vault_secrets() + env = dict( + TF_VAR_project_name=self.project_name, + TF_VAR_project_path=self.vault_project_path, + TF_VAR_secrets=json.dumps(self.vault_secrets), + VAULT_ADDR=self.vault_url, + VAULT_TOKEN=self.vault_token, + ) + self.terraform_backend == TERRAFORM_BACKEND_TFC and env.update( + TF_VAR_terraform_cloud_token=self.terraform_cloud_token + ) + self.run_terraform("vault", env) + + def get_terraform_module_params(self, module_name, env): + """Return Terraform parameters for the given module.""" + return ( + Path(__file__).parent.parent / "terraform" / module_name, + self.logs_dir / self.service_slug / "terraform" / module_name, + terraform_dir := self.terraform_dir / self.service_slug / module_name, + { + **env, + "PATH": os.environ.get("PATH"), + "TF_DATA_DIR": str((terraform_dir / "data").resolve()), + "TF_LOG": "INFO", + }, + ) + + def run_terraform_init(self, cwd, env, logs_dir, state_path): + """Run Terraform init.""" + init_log_path = logs_dir / "init.log" + init_stdout_path = logs_dir / "init-stdout.log" + init_stderr_path = logs_dir / "init-stderr.log" + init_process = subprocess.run( + [ + "terraform", + "init", + "-backend-config", + f"path={state_path.resolve()}", + "-input=false", + "-no-color", + ], + capture_output=True, + cwd=cwd, + env=dict(**env, TF_LOG_PATH=str(init_log_path.resolve())), + text=True, + ) + init_stdout_path.write_text(init_process.stdout) + if init_process.returncode != 0: + init_stderr_path.write_text(init_process.stderr) + click.echo( + error( + "Terraform init failed " + f"(check {init_stderr_path} and {init_log_path})" + ) + ) + raise BootstrapError + + def run_terraform_apply(self, cwd, env, logs_dir): + """Run Terraform apply.""" + apply_log_path = logs_dir / "apply.log" + apply_stdout_path = logs_dir / "apply-stdout.log" + apply_stderr_path = logs_dir / "apply-stderr.log" + apply_process = subprocess.run( + ["terraform", "apply", "-auto-approve", "-input=false", "-no-color"], + capture_output=True, + cwd=cwd, + env=dict(**env, TF_LOG_PATH=str(apply_log_path.resolve())), + text=True, + ) + apply_stdout_path.write_text(apply_process.stdout) + if apply_process.returncode != 0: + apply_stderr_path.write_text(apply_process.stderr) + click.echo( + error( + "Terraform apply failed " + f"(check {apply_stderr_path} and {apply_log_path})" + ) + ) + self.reset_terraform() + raise BootstrapError + + def run_terraform_destroy(self, cwd, env, logs_dir): + """Run Terraform destroy.""" + destroy_log_path = logs_dir / "destroy.log" + destroy_stdout_path = logs_dir / "destroy-stdout.log" + destroy_stderr_path = logs_dir / "destroy-stderr.log" + destroy_process = subprocess.run( + [ + "terraform", + "destroy", + "-auto-approve", + "-input=false", + "-no-color", + ], + capture_output=True, + cwd=cwd, + env=dict(**env, TF_LOG_PATH=str(destroy_log_path.resolve())), + text=True, + ) + destroy_stdout_path.write_text(destroy_process.stdout) + if destroy_process.returncode != 0: + destroy_stderr_path.write_text(destroy_process.stderr) + click.echo( + error( + "Terraform destroy failed " + f"(check {destroy_stderr_path} and {destroy_log_path})" + ) + ) + raise BootstrapError + + def get_terraform_outputs(self, cwd, env, outputs): + """Get Terraform outputs.""" + return { + output_name: subprocess.run( + ["terraform", "output", "-raw", output_name], + capture_output=True, + cwd=cwd, + env=env, + text=True, + ).stdout + for output_name in outputs + } + + def reset_terraform(self): + """Destroy all Terraform modules resources.""" + for module_name, env in self.terraform_run_modules: + click.echo(warning(f"Destroying Terraform {module_name} resources.")) + cwd, logs_dir, _terraform_dir, env = self.get_terraform_module_params( + module_name, env + ) + self.run_terraform_destroy(cwd, env, logs_dir) + + def run_terraform(self, module_name, env, outputs=None): + """Initialize the Terraform controlled resources.""" + self.terraform_run_modules.append((module_name, env)) + cwd, logs_dir, terraform_dir, env = self.get_terraform_module_params( + module_name, env + ) + os.makedirs(terraform_dir, exist_ok=True) + os.makedirs(logs_dir) + self.run_terraform_init(cwd, env, logs_dir, terraform_dir / "terraform.tfstate") + self.run_terraform_apply(cwd, env, logs_dir) + outputs and self.terraform_outputs.update( + {module_name: self.get_terraform_outputs(cwd, env, outputs)} + ) + def init_subrepo(self, service_slug, template_url, **kwargs): """Initialize a subrepo using the given template and options.""" - subrepo_dir = str((Path(SUBREPOS_DIR) / service_slug).resolve()) + subrepo_dir = str((SUBREPOS_DIR / service_slug).resolve()) shutil.rmtree(subrepo_dir, ignore_errors=True) subprocess.run( [ @@ -322,17 +753,21 @@ def init_subrepo(self, service_slug, template_url, **kwargs): "project_url_dev": self.project_url_dev, "project_url_prod": self.project_url_prod, "project_url_stage": self.project_url_stage, + "sentry_org": self.sentry_org, + "sentry_url": self.sentry_url, "service_dir": str((self.service_dir / service_slug).resolve()), "service_slug": service_slug, "terraform_backend": self.terraform_backend, "terraform_cloud_admin_email": self.terraform_cloud_admin_email, "terraform_cloud_hostname": self.terraform_cloud_hostname, - "terraform_cloud_organization": self.terraform_cloud_organization, "terraform_cloud_organization_create": False, + "terraform_cloud_organization": self.terraform_cloud_organization, "terraform_cloud_token": self.terraform_cloud_token, "terraform_dir": str(self.terraform_dir.resolve()), "uid": self.uid, "use_redis": self.use_redis, + "vault_url": self.vault_url, + "vault_token": self.vault_token, **kwargs, } subprocess.run( @@ -360,220 +795,11 @@ def change_output_owner(self): ] ) - def get_gitlab_variables(self): - """Return the GitLab group and project variables.""" - gitlab_group_variables = dict( - BASIC_AUTH_USERNAME=('{value = "%s"}' % self.project_slug), - BASIC_AUTH_PASSWORD=( - '{value = "%s", masked = true}' % secrets.token_urlsafe(12) - ), - ) - gitlab_project_variables = {} - # Sentry and Pact env vars are used by the GitLab CI/CD - self.sentry_org and gitlab_group_variables.update( - SENTRY_ORG='{value = "%s"}' % self.sentry_org, - SENTRY_URL='{value = "%s"}' % self.sentry_url, - SENTRY_AUTH_TOKEN='{value = "%s", masked = true}' % self.sentry_auth_token, - ) - if self.pact_broker_url: - pact_broker_url = self.pact_broker_url - pact_broker_username = self.pact_broker_username - pact_broker_password = self.pact_broker_password - pact_broker_auth_url = re.sub( - r"^(https?)://(.*)$", - rf"\g<1>://{pact_broker_username}:{pact_broker_password}@\g<2>", - pact_broker_url, - ) - gitlab_group_variables.update( - PACT_ENABLED='{value = "true", protected = false}', - PACT_BROKER_BASE_URL=( - '{value = "%s", protected = false}' % pact_broker_url - ), - PACT_BROKER_USERNAME=( - '{value = "%s", protected = false}' % pact_broker_username - ), - PACT_BROKER_PASSWORD=( - '{value = "%s", masked = true, protected = false}' - % pact_broker_password - ), - PACT_BROKER_AUTH_URL=( - '{value = "%s", masked = true, protected = false}' - % pact_broker_auth_url - ), - ) - # TODO: extend after implementing Vault - if self.terraform_backend == TERRAFORM_BACKEND_TFC: - gitlab_group_variables.update( - TFC_TOKEN='{value = "%s", masked = true}' % self.terraform_cloud_token, - ) - if self.subdomain_monitoring: - gitlab_project_variables.update( - GRAFANA_PASSWORD='{value = "%s", masked = true}' - % secrets.token_urlsafe(12), - ) - self.digitalocean_token and gitlab_group_variables.update( - DIGITALOCEAN_TOKEN='{value = "%s", masked = true}' % self.digitalocean_token - ) - if self.deployment_type == DEPLOYMENT_TYPE_OTHER: - gitlab_group_variables.update( - KUBERNETES_CLUSTER_CA_CERTIFICATE='{value = "%s", masked = true}' - % base64.b64encode( - Path(self.kubernetes_cluster_ca_certificate).read_bytes() - ).decode(), - KUBERNETES_HOST='{value = "%s"}' % self.kubernetes_host, - KUBERNETES_TOKEN='{value = "%s", masked = true}' - % self.kubernetes_token, - ) - "s3" in self.media_storage and gitlab_group_variables.update( - S3_ACCESS_ID='{value = "%s", masked = true}' % self.s3_access_id, - S3_SECRET_KEY='{value = "%s", masked = true}' % self.s3_secret_key, - S3_REGION='{value = "%s"}' % self.s3_region, - S3_HOST='{value = "%s"}' % self.s3_host, - ) - if self.media_storage == MEDIA_STORAGE_DIGITALOCEAN_S3: - gitlab_group_variables.update( - S3_HOST='{value = "%s"}' % self.s3_host, - ) - elif self.media_storage == MEDIA_STORAGE_AWS_S3: - gitlab_group_variables.update( - S3_BUCKET_NAME='{value = "%s"}' % self.s3_bucket_name, - ) - return gitlab_group_variables, gitlab_project_variables - - def init_gitlab(self): - """Initialize the GitLab resources.""" - click.echo(info("...creating the GitLab resources with Terraform")) - group_variables, project_variables = self.get_gitlab_variables() - env = dict( - TF_VAR_gitlab_token=self.gitlab_private_token, - TF_VAR_group_maintainers=self.gitlab_group_maintainers, - TF_VAR_group_name=self.project_name, - TF_VAR_group_owners=self.gitlab_group_owners, - TF_VAR_group_slug=self.gitlab_group_slug, - TF_VAR_group_variables="{%s}" - % ", ".join(f"{k} = {v}" for k, v in group_variables.items()), - TF_VAR_local_repository_dir=self.service_dir, - TF_VAR_project_description=( - f'The "{self.project_name}" project {self.service_slug} service.' - ), - TF_VAR_project_name=self.service_slug.title(), - TF_VAR_project_slug=self.service_slug, - TF_VAR_project_variables="{%s}" - % ", ".join(f"{k} = {v}" for k, v in project_variables.items()), - ) - self.run_terraform("gitlab", env) - - def init_terraform_cloud(self): - """Initialize the Terraform Cloud resources.""" - click.echo(info("...creating the Terraform Cloud resources with Terraform")) - stacks_environments = { - k: list(v.keys()) for k, v in self.stacks_environments.items() - } - env = dict( - TF_VAR_admin_email=self.terraform_cloud_admin_email, - TF_VAR_create_organization=self.terraform_cloud_organization_create - and "true" - or "false", - TF_VAR_hostname=self.terraform_cloud_hostname, - TF_VAR_organization_name=self.terraform_cloud_organization, - TF_VAR_project_name=self.project_name, - TF_VAR_project_slug=self.project_slug, - TF_VAR_service_slug="orchestrator", - TF_VAR_stacks=json.dumps(list(stacks_environments.keys())), - TF_VAR_terraform_cloud_token=self.terraform_cloud_token, - ) - self.run_terraform("terraform-cloud", env) - - def run_terraform(self, module_name, env): - """Initialize the Terraform controlled resources.""" - cwd = Path(__file__).parent.parent / "terraform" / module_name - terraform_dir = self.terraform_dir / self.service_slug / module_name - os.makedirs(terraform_dir, exist_ok=True) - env.update( - PATH=os.environ.get("PATH"), - TF_DATA_DIR=str((terraform_dir / "data").resolve()), - TF_LOG="INFO", - ) - state_path = terraform_dir / "state.tfstate" - logs_dir = self.logs_dir / self.service_slug / "terraform" / module_name - os.makedirs(logs_dir) - init_log_path = logs_dir / "init.log" - init_stdout_path = logs_dir / "init-stdout.log" - init_stderr_path = logs_dir / "init-stderr.log" - init_process = subprocess.run( - [ - "terraform", - "init", - "-backend-config", - f"path={state_path.resolve()}", - "-input=false", - "-no-color", - ], - capture_output=True, - cwd=cwd, - env=dict(**env, TF_LOG_PATH=str(init_log_path.resolve())), - text=True, - ) - init_stdout_path.write_text(init_process.stdout) - if init_process.returncode == 0: - apply_log_path = logs_dir / "apply.log" - apply_stdout_path = logs_dir / "apply-stdout.log" - apply_stderr_path = logs_dir / "apply-stderr.log" - apply_process = subprocess.run( - ["terraform", "apply", "-auto-approve", "-input=false", "-no-color"], - capture_output=True, - cwd=cwd, - env=dict(**env, TF_LOG_PATH=str(apply_log_path.resolve())), - text=True, - ) - apply_stdout_path.write_text(apply_process.stdout) - if apply_process.returncode != 0: - apply_stderr_path.write_text(apply_process.stderr) - click.echo( - error( - f"Error applying {module_name} Terraform configuration " - f"(check {apply_stderr_path} and {apply_log_path})" - ) - ) - destroy_log_path = logs_dir / "destroy.log" - destroy_stdout_path = logs_dir / "destroy-stdout.log" - destroy_stderr_path = logs_dir / "destroy-stderr.log" - destroy_process = subprocess.run( - [ - "terraform", - "destroy", - "-auto-approve", - "-input=false", - "-no-color", - ], - capture_output=True, - cwd=cwd, - env=dict(**env, TF_LOG_PATH=str(destroy_log_path.resolve())), - text=True, - ) - destroy_stdout_path.write_text(destroy_process.stdout) - if destroy_process.returncode != 0: - destroy_stderr_path.write_text(destroy_process.stderr) - click.echo( - error( - f"Error performing {module_name} Terraform destroy " - f"(check {destroy_stderr_path} and {destroy_log_path})" - ) - ) - raise BootstrapError - else: - init_stderr_path.write_text(init_process.stderr) - click.echo( - error( - f"Error performing {module_name} Terraform init " - f"(check {init_stderr_path} and {init_log_path})" - ) - ) - raise BootstrapError - def cleanup(self): """Clean up after a successful execution.""" shutil.rmtree(DUMPS_DIR) + shutil.rmtree(SUBREPOS_DIR) + shutil.rmtree(self.terraform_dir) def run(self): """Run the bootstrap.""" @@ -584,6 +810,8 @@ def run(self): self.init_gitlab() if self.terraform_backend == TERRAFORM_BACKEND_TFC: self.init_terraform_cloud() + if self.vault_token: + self.init_vault() frontend_template_url = FRONTEND_TEMPLATE_URLS.get(self.frontend_type) if frontend_template_url: self.init_subrepo( diff --git a/cookiecutter.json b/cookiecutter.json index 6f9fffa3..1efe74b8 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -2,6 +2,7 @@ "project_name": null, "project_slug": "{{ cookiecutter.project_name | slugify() }}", "project_dirname": "{{ cookiecutter.project_slug | slugify(separator='') }}", + "service_slug": "orchestrator", "backend_type": ["django", "none"], "backend_service_slug": "backend", "backend_service_port": "8000", @@ -12,17 +13,17 @@ "terraform_cloud_organization": "", "media_storage": ["digitalocean-s3", "aws-s3", "local", "none"], "pact_enabled": false, - "project_domain": "", + "vault_project_path": "{{ cookiecutter.project_slug }}", "stacks": { "main": { "dev": { - "name": "Development" + "name": "development" }, "stage": { - "name": "Staging" + "name": "staging" }, "prod": { - "name": "Production" + "name": "production" } } }, diff --git a/requirements/common.in b/requirements/common.in index d2f8a351..e05d8978 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -1,5 +1,5 @@ click~=8.1.0 -cookiecutter~=1.7.0 +cookiecutter~=2.1.0 pydantic~=1.9.0 python-slugify~=6.1.0 -validators~=0.18.0 +validators~=0.20.0 diff --git a/requirements/common.txt b/requirements/common.txt index 1290304d..5a443886 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -2,23 +2,23 @@ arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via # -r requirements/common.in # cookiecutter -cookiecutter==1.7.3 +cookiecutter==2.1.1 # via -r requirements/common.in decorator==5.1.1 # via validators idna==3.3 # via requests -jinja2==3.1.1 +jinja2==3.1.2 # via # cookiecutter # jinja2-time @@ -26,28 +26,25 @@ jinja2-time==0.2.0 # via cookiecutter markupsafe==2.1.1 # via jinja2 -poyo==0.5.0 - # via cookiecutter -pydantic==1.9.0 +pydantic==1.9.1 # via -r requirements/common.in python-dateutil==2.8.2 # via arrow -python-slugify==6.1.1 +python-slugify==6.1.2 # via # -r requirements/common.in # cookiecutter -requests==2.27.1 +pyyaml==6.0 + # via cookiecutter +requests==2.28.0 # via cookiecutter six==1.16.0 - # via - # cookiecutter - # python-dateutil - # validators + # via python-dateutil text-unidecode==1.3 # via python-slugify typing-extensions==4.2.0 # via pydantic urllib3==1.26.9 # via requests -validators==0.18.2 +validators==0.20.0 # via -r requirements/common.in diff --git a/requirements/local.in b/requirements/local.in index 8fb91536..86721c7e 100644 --- a/requirements/local.in +++ b/requirements/local.in @@ -1,7 +1,7 @@ -r test.in black~=22.3.0 ipdb~=0.13.0 -pip-tools~=6.5.0 -pre-commit~=2.17.0 -pyupgrade~=2.31.0 +pip-tools~=6.6.0 +pre-commit~=2.19.0 +pyupgrade~=2.34.0 types-python-slugify~=5.0.0 diff --git a/requirements/local.txt b/requirements/local.txt index d0787f9a..ae12214c 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -10,7 +10,7 @@ binaryornot==0.4.4 # via cookiecutter black==22.3.0 # via -r requirements/local.in -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests cfgv==3.3.1 # via pre-commit @@ -18,15 +18,15 @@ chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via # -r requirements/common.in # black # cookiecutter # pip-tools -cookiecutter==1.7.3 +cookiecutter==2.1.1 # via -r requirements/common.in -coverage[toml]==6.3.2 +coverage[toml]==6.4.1 # via -r requirements/test.in decorator==5.1.1 # via @@ -37,7 +37,7 @@ distlib==0.3.4 # via virtualenv executing==0.8.3 # via stack-data -filelock==3.6.0 +filelock==3.7.1 # via virtualenv flake8==4.0.1 # via @@ -45,19 +45,19 @@ flake8==4.0.1 # flake8-bugbear # flake8-docstrings # flake8-isort -flake8-bugbear==22.1.11 +flake8-bugbear==22.4.25 # via -r requirements/test.in flake8-docstrings==1.6.0 # via -r requirements/test.in flake8-isort==4.1.1 # via -r requirements/test.in -identify==2.5.0 +identify==2.5.1 # via pre-commit idna==3.3 # via requests ipdb==0.13.9 # via -r requirements/local.in -ipython==8.2.0 +ipython==8.4.0 # via ipdb isort==5.10.1 # via @@ -65,7 +65,7 @@ isort==5.10.1 # flake8-isort jedi==0.18.1 # via ipython -jinja2==3.1.1 +jinja2==3.1.2 # via # cookiecutter # jinja2-time @@ -77,7 +77,7 @@ matplotlib-inline==0.1.3 # via ipython mccabe==0.6.1 # via flake8 -mypy==0.942 +mypy==0.961 # via -r requirements/test.in mypy-extensions==0.4.3 # via @@ -95,15 +95,13 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.5.1 +pip-tools==6.6.2 # via -r requirements/local.in platformdirs==2.5.2 # via # black # virtualenv -poyo==0.5.0 - # via cookiecutter -pre-commit==2.17.0 +pre-commit==2.19.0 # via -r requirements/local.in prompt-toolkit==3.0.29 # via ipython @@ -113,7 +111,7 @@ pure-eval==0.2.2 # via stack-data pycodestyle==2.8.0 # via flake8 -pydantic==1.9.0 +pydantic==1.9.1 # via -r requirements/common.in pydocstyle==6.1.1 # via flake8-docstrings @@ -123,22 +121,22 @@ pygments==2.12.0 # via ipython python-dateutil==2.8.2 # via arrow -python-slugify==6.1.1 +python-slugify==6.1.2 # via # -r requirements/common.in # cookiecutter -pyupgrade==2.31.1 +pyupgrade==2.34.0 # via -r requirements/local.in pyyaml==6.0 - # via pre-commit -requests==2.27.1 + # via + # cookiecutter + # pre-commit +requests==2.28.0 # via cookiecutter six==1.16.0 # via # asttokens - # cookiecutter # python-dateutil - # validators # virtualenv snowballstemmer==2.2.0 # via pydocstyle @@ -160,7 +158,7 @@ tomli==2.0.1 # coverage # mypy # pep517 -traitlets==5.1.1 +traitlets==5.2.2.post1 # via # ipython # matplotlib-inline @@ -172,7 +170,7 @@ typing-extensions==4.2.0 # pydantic urllib3==1.26.9 # via requests -validators==0.18.2 +validators==0.20.0 # via -r requirements/common.in virtualenv==20.14.1 # via pre-commit diff --git a/requirements/test.in b/requirements/test.in index 0a3c21a0..c735f8c3 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -1,8 +1,8 @@ -r common.in -coverage[toml]~=6.3.0 -flake8-bugbear~=22.1.0 +coverage[toml]~=6.4.0 +flake8-bugbear~=22.4.0 flake8-docstrings~=1.6.0 flake8-isort~=4.1.0 flake8~=4.0.0 isort~=5.10.0 -mypy~=0.931 +mypy~=0.961 diff --git a/requirements/test.txt b/requirements/test.txt index 220d1eed..132dbd06 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,19 +4,19 @@ attrs==21.4.0 # via flake8-bugbear binaryornot==0.4.4 # via cookiecutter -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests chardet==4.0.0 # via binaryornot charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via # -r requirements/common.in # cookiecutter -cookiecutter==1.7.3 +cookiecutter==2.1.1 # via -r requirements/common.in -coverage[toml]==6.3.2 +coverage[toml]==6.4.1 # via -r requirements/test.in decorator==5.1.1 # via validators @@ -26,7 +26,7 @@ flake8==4.0.1 # flake8-bugbear # flake8-docstrings # flake8-isort -flake8-bugbear==22.1.11 +flake8-bugbear==22.4.25 # via -r requirements/test.in flake8-docstrings==1.6.0 # via -r requirements/test.in @@ -38,7 +38,7 @@ isort==5.10.1 # via # -r requirements/test.in # flake8-isort -jinja2==3.1.1 +jinja2==3.1.2 # via # cookiecutter # jinja2-time @@ -48,15 +48,13 @@ markupsafe==2.1.1 # via jinja2 mccabe==0.6.1 # via flake8 -mypy==0.942 +mypy==0.961 # via -r requirements/test.in mypy-extensions==0.4.3 # via mypy -poyo==0.5.0 - # via cookiecutter pycodestyle==2.8.0 # via flake8 -pydantic==1.9.0 +pydantic==1.9.1 # via -r requirements/common.in pydocstyle==6.1.1 # via flake8-docstrings @@ -64,17 +62,16 @@ pyflakes==2.4.0 # via flake8 python-dateutil==2.8.2 # via arrow -python-slugify==6.1.1 +python-slugify==6.1.2 # via # -r requirements/common.in # cookiecutter -requests==2.27.1 +pyyaml==6.0 + # via cookiecutter +requests==2.28.0 # via cookiecutter six==1.16.0 - # via - # cookiecutter - # python-dateutil - # validators + # via python-dateutil snowballstemmer==2.2.0 # via pydocstyle testfixtures==6.18.5 @@ -91,5 +88,5 @@ typing-extensions==4.2.0 # pydantic urllib3==1.26.9 # via requests -validators==0.18.2 +validators==0.20.0 # via -r requirements/common.in diff --git a/start.py b/start.py index 04434bed..55521281 100755 --- a/start.py +++ b/start.py @@ -53,6 +53,8 @@ default=None, ) @click.option("--terraform-cloud-admin-email") +@click.option("--vault-token") +@click.option("--vault-url") @click.option("--digitalocean-token") @click.option( "--kubernetes-cluster-ca-certificate", @@ -110,6 +112,7 @@ @click.option("--s3-access-id") @click.option("--s3-secret-key") @click.option("--s3-bucket-name") +@click.option("--gitlab-url") @click.option("--gitlab-private-token", envvar=GITLAB_TOKEN_ENV_VAR) @click.option("--gitlab-group-slug") @click.option("--gitlab-group-owners") diff --git a/terraform/gitlab/main.tf b/terraform/gitlab/main.tf index 2b23787a..17fb2e5a 100644 --- a/terraform/gitlab/main.tf +++ b/terraform/gitlab/main.tf @@ -34,7 +34,7 @@ terraform { required_providers { gitlab = { source = "gitlabhq/gitlab" - version = "~> 3.13" + version = "~> 3.16" } } } @@ -184,6 +184,8 @@ resource "gitlab_group_variable" "vars" { } resource "gitlab_group_variable" "registry_password" { + count = var.vault_enabled ? 0 : 1 + group = data.gitlab_group.group.id key = "REGISTRY_PASSWORD" value = gitlab_deploy_token.regcred.token @@ -192,6 +194,8 @@ resource "gitlab_group_variable" "registry_password" { } resource "gitlab_group_variable" "registry_username" { + count = var.vault_enabled ? 0 : 1 + group = data.gitlab_group.group.id key = "REGISTRY_USERNAME" value = gitlab_deploy_token.regcred.username diff --git a/terraform/gitlab/outputs.tf b/terraform/gitlab/outputs.tf new file mode 100644 index 00000000..3c46c636 --- /dev/null +++ b/terraform/gitlab/outputs.tf @@ -0,0 +1,11 @@ +output "registry_password" { + description = "The password to access the GitLab images registry." + sensitive = true + value = gitlab_deploy_token.regcred.token +} + +output "registry_username" { + description = "The username to access the GitLab images registry." + sensitive = true + value = gitlab_deploy_token.regcred.username +} diff --git a/terraform/gitlab/variables.tf b/terraform/gitlab/variables.tf index 3dbd0c1f..606790d2 100644 --- a/terraform/gitlab/variables.tf +++ b/terraform/gitlab/variables.tf @@ -62,3 +62,9 @@ variable "project_variables" { type = map(map(any)) default = {} } + +variable "vault_enabled" { + description = "Tell if Vault is enabled." + type = bool + default = false +} diff --git a/terraform/terraform-cloud/main.tf b/terraform/terraform-cloud/main.tf index e123d89a..1e7a3b0c 100644 --- a/terraform/terraform-cloud/main.tf +++ b/terraform/terraform-cloud/main.tf @@ -13,8 +13,8 @@ locals { tags = [ "project:${var.project_slug}", "service:${var.service_slug}", - "stage:${stage}", "stack:${stack}", + "stage:${stage}", ] } ] @@ -26,10 +26,10 @@ locals { name = "${var.project_slug}_${var.service_slug}_environment_${env}" description = "${var.project_name} project, ${var.service_slug} service, ${env} environment" tags = [ + "env:${env}", "project:${var.project_slug}", "service:${var.service_slug}", "stage:environment", - "env:${env}", ] } ] @@ -43,7 +43,7 @@ terraform { required_providers { tfe = { source = "hashicorp/tfe" - version = "~> 0.30" + version = "~> 0.32" } } } diff --git a/terraform/terraform-cloud/variables.tf b/terraform/terraform-cloud/variables.tf index ee4da861..0b433609 100644 --- a/terraform/terraform-cloud/variables.tf +++ b/terraform/terraform-cloud/variables.tf @@ -11,9 +11,8 @@ variable "create_organization" { } variable "environments" { - description = "The list of environment slugs." + description = "The list of environments slugs." type = list(string) - default = ["dev", "stage", "prod"] } variable "hostname" { @@ -43,7 +42,7 @@ variable "service_slug" { } variable "stacks" { - description = "The list of stack slugs." + description = "The list of stacks slugs." type = list(string) } diff --git a/terraform/vault-admin/main.tf b/terraform/vault-admin/main.tf new file mode 100644 index 00000000..ae179fc1 --- /dev/null +++ b/terraform/vault-admin/main.tf @@ -0,0 +1,192 @@ +terraform { + required_providers { + vault = { + source = "hashicorp/vault" + version = "3.7.0" + } + } +} + +provider "vault" { +} + +/* GitLab JWT Auth */ + +resource "vault_jwt_auth_backend" "gitlab_jwt" { + description = "Demonstration of the Terraform JWT auth backend" + path = "gitlab-jwt-${var.project_path}" + jwks_url = format("%s/-/jwks", trimsuffix(var.gitlab_url, "/")) + bound_issuer = var.gitlab_url +} + +resource "vault_policy" "gitlab_jwt_stacks" { + for_each = var.stacks_environments + + name = "gitlab-jwt-${var.project_path}-stacks-${each.key}" + + policy = < keys(v) }) + + name = "gitlab-jwt-${var.project_path}-envs-${each.key}" + + policy = < -This is the "{{ cookiecutter.project_name }}" orchestrator. +This is the "{{ cookiecutter.project_name }}" {{ cookiecutter.service_slug }}. ## Index @@ -27,7 +27,7 @@ This is the "{{ cookiecutter.project_name }}" orchestrator. ## Provisioning -The first run is manual, made from [GitLab Pipeline](https://gitlab.com/{{ cookiecutter.project_slug }}/orchestrator/-/pipelines/new). +The first run is manual, made from [GitLab Pipeline](https://gitlab.com/{{ cookiecutter.project_slug }}/{{ cookiecutter.service_slug }}/-/pipelines/new). To create all the terraform resources, run the pipeline with the following variable: @@ -84,10 +84,10 @@ This section explains the steps you need to clone and work with this project. #### Clone -Clone the orchestrator and services repositories: +Clone the {{ cookiecutter.service_slug }} and services repositories: ```console -git clone git@gitlab.com:{{ cookiecutter.project_slug }}/orchestrator.git {{ cookiecutter.project_dirname }} +git clone git@gitlab.com:{{ cookiecutter.project_slug }}/{{ cookiecutter.service_slug }}.git {{ cookiecutter.project_dirname }} cd {{ cookiecutter.project_dirname }}{% if cookiecutter.backend_type != 'none' %} git clone -b develop git@gitlab.com:{{ cookiecutter.project_slug }}/{{ cookiecutter.backend_service_slug }}.git{% endif %}{% if cookiecutter.frontend_type != 'none' %} git clone -b develop git@gitlab.com:{{ cookiecutter.project_slug }}/{{ cookiecutter.frontend_service_slug }}.git{% endif %} @@ -98,7 +98,7 @@ cd .. ### Environment variables -In order for the project to run correctly, a number of environment variables must be set in an `.env` file inside the orchestrator directory. For ease of use, a `.env_template` template is provided. +In order for the project to run correctly, a number of environment variables must be set in an `.env` file inside the {{ cookiecutter.service_slug }} directory. For ease of use, a `.env_template` template is provided. Enter the newly created **project** directory and create the `.env` file copying from `.env_template`: @@ -109,7 +109,7 @@ $ cp .env_template .env ### Docker -All the following Docker commands are supposed to be run from the orchestrator directory. +All the following Docker commands are supposed to be run from the {{ cookiecutter.service_slug }} directory. #### Build @@ -181,11 +181,10 @@ $ make rebuild s=backend Import the `traefik/20tab.crt` file in your browser to have a trusted ssl certificate: -#### Firefox +#### Firefox -- Settings > Privacy & Security > Manage Certificates > View Certificates... > Authorities > Import +- Settings > Privacy & Security > Manage Certificates > View Certificates... > Authorities > Import #### Chrome -- Settings > Security > Certificates > Authorities > Import - +- Settings > Security > Certificates > Authorities > Import diff --git a/{{cookiecutter.project_dirname}}/docker-compose.yaml b/{{cookiecutter.project_dirname}}/docker-compose.yaml index 019031c4..eb96c479 100644 --- a/{{cookiecutter.project_dirname}}/docker-compose.yaml +++ b/{{cookiecutter.project_dirname}}/docker-compose.yaml @@ -69,4 +69,3 @@ services: volumes: pg_data: {} - diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy.sh b/{{cookiecutter.project_dirname}}/scripts/deploy.sh new file mode 100755 index 00000000..88e26cea --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy.sh @@ -0,0 +1,6 @@ +#!/bin/sh -e + +# init.sh must be sourced to let it export env vars +. ${PROJECT_DIR}/scripts/deploy/init.sh + +sh ${PROJECT_DIR}/scripts/deploy/terraform.sh ${@} diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy/gitlab.sh b/{{cookiecutter.project_dirname}}/scripts/deploy/gitlab.sh new file mode 100755 index 00000000..8a7cf78c --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy/gitlab.sh @@ -0,0 +1,32 @@ +#!/bin/sh -e + +# If TF_USERNAME is unset then default to GITLAB_USER_LOGIN +TF_USERNAME="${TF_USERNAME:-${GITLAB_USER_LOGIN}}" +# If TF_PASSWORD is unset then default to gitlab-ci-token/CI_JOB_TOKEN +if [ -z "${TF_PASSWORD}" ]; then +TF_USERNAME="gitlab-ci-token" +TF_PASSWORD="${CI_JOB_TOKEN}" +fi +# If TF_ADDRESS is unset but TF_STATE_NAME is provided, then default to GitLab backend in current project +if [ -n "${TF_STATE_NAME}" ]; then +TF_ADDRESS="${TF_ADDRESS:-${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}}" +fi +# Set variables for the HTTP backend to default to TF_* values +export TF_HTTP_ADDRESS="${TF_HTTP_ADDRESS:-${TF_ADDRESS}}" +export TF_HTTP_LOCK_ADDRESS="${TF_HTTP_LOCK_ADDRESS:-${TF_ADDRESS}/lock}" +export TF_HTTP_LOCK_METHOD="${TF_HTTP_LOCK_METHOD:-POST}" +export TF_HTTP_UNLOCK_ADDRESS="${TF_HTTP_UNLOCK_ADDRESS:-${TF_ADDRESS}/lock}" +export TF_HTTP_UNLOCK_METHOD="${TF_HTTP_UNLOCK_METHOD:-DELETE}" +export TF_HTTP_USERNAME="${TF_HTTP_USERNAME:-${TF_USERNAME}}" +export TF_HTTP_PASSWORD="${TF_HTTP_PASSWORD:-${TF_PASSWORD}}" +export TF_HTTP_RETRY_WAIT_MIN="${TF_HTTP_RETRY_WAIT_MIN:-5}" +# Expose Gitlab specific variables to terraform since no -tf-var is available +# Usable in the .tf file as variable "CI_JOB_ID" { type = string } etc +export TF_VAR_CI_JOB_ID="${TF_VAR_CI_JOB_ID:-${CI_JOB_ID}}" +export TF_VAR_CI_COMMIT_SHA="${TF_VAR_CI_COMMIT_SHA:-${CI_COMMIT_SHA}}" +export TF_VAR_CI_JOB_STAGE="${TF_VAR_CI_JOB_STAGE:-${CI_JOB_STAGE}}" +export TF_VAR_CI_PROJECT_ID="${TF_VAR_CI_PROJECT_ID:-${CI_PROJECT_ID}}" +export TF_VAR_CI_PROJECT_NAME="${TF_VAR_CI_PROJECT_NAME:-${CI_PROJECT_NAME}}" +export TF_VAR_CI_PROJECT_NAMESPACE="${TF_VAR_CI_PROJECT_NAMESPACE:-${CI_PROJECT_NAMESPACE}}" +export TF_VAR_CI_PROJECT_PATH="${TF_VAR_CI_PROJECT_PATH:-${CI_PROJECT_PATH}}" +export TF_VAR_CI_PROJECT_URL="${TF_VAR_CI_PROJECT_URL:-${CI_PROJECT_URL}}" diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy/init.sh b/{{cookiecutter.project_dirname}}/scripts/deploy/init.sh new file mode 100755 index 00000000..35485075 --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy/init.sh @@ -0,0 +1,30 @@ +#!/bin/sh -e + +export TF_VAR_env_slug="${ENV_SLUG}" +export TF_VAR_project_slug="${PROJECT_SLUG}" +export TF_VAR_stack_slug="${STACK_SLUG}" + +var_file="${TERRAFORM_VARS_DIR}/.tfvars" + +if [ "${TERRAFORM_EXTRA_VAR_FILE}" != "" ]; then + extra_var_file="${TERRAFORM_VARS_DIR}/${TERRAFORM_EXTRA_VAR_FILE}" + touch ${extra_var_file} + var_files="-var-file=${var_file} -var-file=${extra_var_file}" +else + var_files="-var-file=${var_file}" +fi + +if [ "${VAULT_ADDR}" != "" ]; then + . ${PROJECT_DIR}/scripts/deploy/vault.sh +fi + +export TERRAFORM_VAR_FILE_ARGS=${var_files} + +case "${TERRAFORM_BACKEND}" in + "gitlab") + . ${PROJECT_DIR}/scripts/deploy/gitlab.sh + ;; + "terraform-cloud") + . ${PROJECT_DIR}/scripts/deploy/terraform-cloud.sh + ;; +esac diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy/terraform-cloud.sh b/{{cookiecutter.project_dirname}}/scripts/deploy/terraform-cloud.sh new file mode 100755 index 00000000..515e6ca1 --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy/terraform-cloud.sh @@ -0,0 +1,12 @@ +#!/bin/sh -e + +export TF_CLI_CONFIG_FILE="${TF_ROOT}/cloud.tfc" +cat << EOF > ${TF_CLI_CONFIG_FILE} +{ + "credentials": { + "app.terraform.io": { + "token": "${TFC_TOKEN}" + } + } +} +EOF diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy/terraform.sh b/{{cookiecutter.project_dirname}}/scripts/deploy/terraform.sh new file mode 100755 index 00000000..1bd1a45a --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy/terraform.sh @@ -0,0 +1,67 @@ +#!/bin/sh -e + +if [ "${DEBUG_OUTPUT}" == "true" ]; then + set -x +fi + +plan_cache="plan.cache" +plan_json="plan.json" + +JQ_PLAN=' + ( + [.resource_changes[]?.change.actions?] | flatten + ) | { + "create":(map(select(.=="create")) | length), + "update":(map(select(.=="update")) | length), + "delete":(map(select(.=="delete")) | length) + } +' + +# Use terraform automation mode (will remove some verbose unneeded messages) +export TF_IN_AUTOMATION=true + +init() { + cd ${TF_ROOT} + if [ "${TERRAFORM_BACKEND}" == "terraform-cloud" ]; then + terraform init "${@}" -input=false + else + terraform init "${@}" -input=false -reconfigure + fi +} + +case "${1}" in + "apply") + init + terraform "${@}" -input=false "${plan_cache}" + ;; + "destroy") + init + terraform "${@}" ${TERRAFORM_VAR_FILE_ARGS} -auto-approve + ;; + "fmt") + terraform "${@}" -check -diff -recursive + ;; + "init") + # shift argument list „one to the left“ to not call 'terraform init init' + shift + init "${@}" + ;; + "plan") + init + terraform "${@}" ${TERRAFORM_VAR_FILE_ARGS} -input=false -out="${plan_cache}" + ;; + "plan-json") + init + terraform plan ${TERRAFORM_VAR_FILE_ARGS} -input=false -out="${plan_cache}" + terraform show -json "${plan_cache}" | \ + jq -r "${JQ_PLAN}" \ + > "${plan_json}" + ;; + "validate") + init -backend=false + terraform "${@}" + ;; + *) + terraform "${@}" + ;; +esac diff --git a/{{cookiecutter.project_dirname}}/scripts/deploy/vault.sh b/{{cookiecutter.project_dirname}}/scripts/deploy/vault.sh new file mode 100755 index 00000000..50e19f81 --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/deploy/vault.sh @@ -0,0 +1,29 @@ +#!/bin/sh -e + +curl https://releases.hashicorp.com/vault/${VAULT_VERSION:=1.11.0}/vault_${VAULT_VERSION}_linux_386.zip --output vault.zip +unzip vault.zip + +load_secrets() +{ + secrets_prefix=$1 + secrets_slug=$2 + export VAULT_TOKEN="$(./vault write -field=token auth/gitlab-jwt-${VAULT_PROJECT_PATH}/login role=gitlab-jwt-${VAULT_PROJECT_PATH}-${secrets_prefix}-${secrets_slug} jwt=${CI_JOB_JWT_V2})" + for secret_name in $3 + do + secret_var_file=${TERRAFORM_VARS_DIR}/${secret_name}.json + (./vault kv get -format='json' -field=data ${VAULT_PROJECT_PATH}/${secrets_prefix}/${secrets_slug}/${secret_name} 2> /dev/null || echo {}) > ${secret_var_file} + var_files="${var_files} -var-file=${secret_var_file}" + done +} + +if [ "${STACK_SLUG}" != "" ] && [ "${VAULT_STACK_SECRETS}" != "" ]; then + load_secrets "stacks" ${STACK_SLUG} "${VAULT_STACK_SECRETS}" +fi + +if [ "${ENV_SLUG}" != "" ] && [ "${VAULT_ENV_SECRETS}" != "" ]; then + load_secrets "envs" ${ENV_SLUG} "${VAULT_ENV_SECRETS}" +fi + +if [ "${TERRAFORM_BACKEND}" == "terraform-cloud" ]; then + export TFC_TOKEN="$(./vault read -field=token ${PROJECT_SLUG}-tfc/creds/default)" +fi diff --git a/{{cookiecutter.project_dirname}}/scripts/terraform.sh b/{{cookiecutter.project_dirname}}/scripts/terraform.sh deleted file mode 100755 index 26c4dab6..00000000 --- a/{{cookiecutter.project_dirname}}/scripts/terraform.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh -e - -if [ "${DEBUG_OUTPUT}" == "true" ]; then - set -x -fi - -plan_cache="plan.cache" -plan_json="plan.json" -var_file="${TERRAFORM_VARS_DIR}/.tfvars" - -JQ_PLAN=' - ( - [.resource_changes[]?.change.actions?] | flatten - ) | { - "create":(map(select(.=="create")) | length), - "update":(map(select(.=="update")) | length), - "delete":(map(select(.=="delete")) | length) - } -' - -case "${TERRAFORM_BACKEND}" in - "gitlab") - # If TF_USERNAME is unset then default to GITLAB_USER_LOGIN - TF_USERNAME="${TF_USERNAME:-${GITLAB_USER_LOGIN}}" - # If TF_PASSWORD is unset then default to gitlab-ci-token/CI_JOB_TOKEN - if [ -z "${TF_PASSWORD}" ]; then - TF_USERNAME="gitlab-ci-token" - TF_PASSWORD="${CI_JOB_TOKEN}" - fi - # If TF_ADDRESS is unset but TF_STATE_NAME is provided, then default to GitLab backend in current project - if [ -n "${TF_STATE_NAME}" ]; then - TF_ADDRESS="${TF_ADDRESS:-${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${TF_STATE_NAME}}" - fi - # Set variables for the HTTP backend to default to TF_* values - export TF_HTTP_ADDRESS="${TF_HTTP_ADDRESS:-${TF_ADDRESS}}" - export TF_HTTP_LOCK_ADDRESS="${TF_HTTP_LOCK_ADDRESS:-${TF_ADDRESS}/lock}" - export TF_HTTP_LOCK_METHOD="${TF_HTTP_LOCK_METHOD:-POST}" - export TF_HTTP_UNLOCK_ADDRESS="${TF_HTTP_UNLOCK_ADDRESS:-${TF_ADDRESS}/lock}" - export TF_HTTP_UNLOCK_METHOD="${TF_HTTP_UNLOCK_METHOD:-DELETE}" - export TF_HTTP_USERNAME="${TF_HTTP_USERNAME:-${TF_USERNAME}}" - export TF_HTTP_PASSWORD="${TF_HTTP_PASSWORD:-${TF_PASSWORD}}" - export TF_HTTP_RETRY_WAIT_MIN="${TF_HTTP_RETRY_WAIT_MIN:-5}" - # Expose Gitlab specific variables to terraform since no -tf-var is available - # Usable in the .tf file as variable "CI_JOB_ID" { type = string } etc - export TF_VAR_CI_JOB_ID="${TF_VAR_CI_JOB_ID:-${CI_JOB_ID}}" - export TF_VAR_CI_COMMIT_SHA="${TF_VAR_CI_COMMIT_SHA:-${CI_COMMIT_SHA}}" - export TF_VAR_CI_JOB_STAGE="${TF_VAR_CI_JOB_STAGE:-${CI_JOB_STAGE}}" - export TF_VAR_CI_PROJECT_ID="${TF_VAR_CI_PROJECT_ID:-${CI_PROJECT_ID}}" - export TF_VAR_CI_PROJECT_NAME="${TF_VAR_CI_PROJECT_NAME:-${CI_PROJECT_NAME}}" - export TF_VAR_CI_PROJECT_NAMESPACE="${TF_VAR_CI_PROJECT_NAMESPACE:-${CI_PROJECT_NAMESPACE}}" - export TF_VAR_CI_PROJECT_PATH="${TF_VAR_CI_PROJECT_PATH:-${CI_PROJECT_PATH}}" - ;; - "terraform-cloud") - export TF_CLI_CONFIG_FILE="${TF_ROOT}/cloud.tfc" - cat << EOF > ${TF_CLI_CONFIG_FILE} -{ - "credentials": { - "app.terraform.io": { - "token": "${TFC_TOKEN}" - } - } -} -EOF - ;; -esac - -if [ "${TERRAFORM_EXTRA_VAR_FILE}" != "" ]; then - extra_var_file="${TERRAFORM_VARS_DIR}/${TERRAFORM_EXTRA_VAR_FILE}" - touch ${extra_var_file} - var_files="-var-file=${var_file} -var-file=${extra_var_file}" -else - var_files="-var-file=${var_file}" -fi - -# Use terraform automation mode (will remove some verbose unneeded messages) -export TF_IN_AUTOMATION=true - -init() { - cd ${TF_ROOT} - if [ "${TERRAFORM_BACKEND}" == "terraform-cloud" ]; then - terraform init "${@}" -input=false - else - terraform init "${@}" -input=false -reconfigure - fi -} - -case "${1}" in - "apply") - init - terraform "${@}" -input=false "${plan_cache}" - ;; - "destroy") - init - terraform "${@}" ${var_files} -auto-approve - ;; - "fmt") - terraform "${@}" -check -diff -recursive - ;; - "init") - # shift argument list „one to the left“ to not call 'terraform init init' - shift - init "${@}" - ;; - "plan") - init - terraform "${@}" ${var_files} -input=false -out="${plan_cache}" - ;; - "plan-json") - init - terraform plan ${var_files} -input=false -out="${plan_cache}" - terraform show -json "${plan_cache}" | \ - jq -r "${JQ_PLAN}" \ - > "${plan_json}" - ;; - "validate") - init -backend=false - terraform "${@}" - ;; - *) - terraform "${@}" - ;; -esac diff --git a/{{cookiecutter.project_dirname}}/terraform/base/digitalocean-k8s/main.tf b/{{cookiecutter.project_dirname}}/terraform/base/digitalocean-k8s/main.tf index 892607ce..3d7777ab 100644 --- a/{{cookiecutter.project_dirname}}/terraform/base/digitalocean-k8s/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/base/digitalocean-k8s/main.tf @@ -24,7 +24,7 @@ terraform { required_providers { digitalocean = { source = "digitalocean/digitalocean" - version = "~> 2.0" + version = "~> 2.21" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/cluster/digitalocean-k8s/main.tf b/{{cookiecutter.project_dirname}}/terraform/cluster/digitalocean-k8s/main.tf index adfb9c42..0b18ba43 100644 --- a/{{cookiecutter.project_dirname}}/terraform/cluster/digitalocean-k8s/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/cluster/digitalocean-k8s/main.tf @@ -6,15 +6,15 @@ terraform { required_providers { digitalocean = { source = "digitalocean/digitalocean" - version = "~> 2.0" + version = "~> 2.21" } helm = { source = "hashicorp/helm" - version = "2.5.0" + version = "~> 2.6" } kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/cluster/modules/kubernetes/traefik/main.tf b/{{cookiecutter.project_dirname}}/terraform/cluster/modules/kubernetes/traefik/main.tf index 69ae988f..c3529c19 100644 --- a/{{cookiecutter.project_dirname}}/terraform/cluster/modules/kubernetes/traefik/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/cluster/modules/kubernetes/traefik/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { helm = { source = "hashicorp/helm" - version = "2.5.0" + version = "~> 2.6" } kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/cluster/other-k8s/main.tf b/{{cookiecutter.project_dirname}}/terraform/cluster/other-k8s/main.tf index 4d145233..57ef8965 100644 --- a/{{cookiecutter.project_dirname}}/terraform/cluster/other-k8s/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/cluster/other-k8s/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { helm = { source = "hashicorp/helm" - version = "2.5.0" + version = "~> 2.6" } kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/digitalocean-k8s/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/digitalocean-k8s/main.tf index 2dd1a4a7..3b33b239 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/digitalocean-k8s/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/digitalocean-k8s/main.tf @@ -15,7 +15,8 @@ locals { var.s3_region != "", var.s3_access_id != "", var.s3_secret_key != "", - local.s3_host != "" || local.s3_bucket_name != "", + local.s3_bucket_name != "", + local.s3_host != "", ] ) } @@ -24,11 +25,11 @@ terraform { required_providers { digitalocean = { source = "digitalocean/digitalocean" - version = "~> 2.0" + version = "~> 2.21" } kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } @@ -188,7 +189,7 @@ module "routing" { /* Routing Metrics */ module "metrics" { - count = var.stack_slug == "main" ? 1 : 0 + count = var.stack_slug == "main" && var.env_slug == "prod" ? 1 : 0 source = "../modules/kubernetes/metrics" diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/database-dump-cronjob/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/database-dump-cronjob/main.tf index a59303be..70531f3f 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/database-dump-cronjob/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/database-dump-cronjob/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/metrics/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/metrics/main.tf index 7e32d24c..86c6de0b 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/metrics/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/metrics/main.tf @@ -11,7 +11,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/monitoring/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/monitoring/main.tf index a2fc1ba5..948dc9f7 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/monitoring/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/monitoring/main.tf @@ -6,11 +6,11 @@ terraform { required_providers { helm = { source = "hashicorp/helm" - version = "2.5.0" + version = "~> 2.6" } kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/postgres/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/postgres/main.tf index bdb85264..eac58153 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/postgres/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/postgres/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } random = { source = "hashicorp/random" - version = "~> 3.1" + version = "~> 3.3" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/redis/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/redis/main.tf index aee3d7dc..546baa43 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/redis/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/redis/main.tf @@ -2,11 +2,11 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } random = { source = "hashicorp/random" - version = "~> 3.1" + version = "~> 3.3" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/routing/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/routing/main.tf index d9538791..1e664840 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/routing/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/modules/kubernetes/routing/main.tf @@ -25,7 +25,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } } } diff --git a/{{cookiecutter.project_dirname}}/terraform/environment/other-k8s/main.tf b/{{cookiecutter.project_dirname}}/terraform/environment/other-k8s/main.tf index 9b65de22..9c50f66d 100644 --- a/{{cookiecutter.project_dirname}}/terraform/environment/other-k8s/main.tf +++ b/{{cookiecutter.project_dirname}}/terraform/environment/other-k8s/main.tf @@ -8,7 +8,7 @@ locals { var.s3_region != "", var.s3_access_id != "", var.s3_secret_key != "", - var.s3_host != "" || var.s3_bucket_name != "", + var.s3_bucket_name != "", ] ) } @@ -17,11 +17,11 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "2.9.0" + version = "~> 2.12" } random = { source = "hashicorp/random" - version = "~> 3.1" + version = "~> 3.3" } } } @@ -117,7 +117,7 @@ module "routing" { /* Metrics */ module "metrics" { - count = var.stack_slug == "main" ? 1 : 0 + count = var.stack_slug == "main" && var.env_slug == "prod" ? 1 : 0 source = "../modules/kubernetes/metrics" diff --git a/{{cookiecutter.project_dirname}}/traefik/conf/static.yaml b/{{cookiecutter.project_dirname}}/traefik/conf/static.yaml index 561d5ead..6458509c 100644 --- a/{{cookiecutter.project_dirname}}/traefik/conf/static.yaml +++ b/{{cookiecutter.project_dirname}}/traefik/conf/static.yaml @@ -8,11 +8,11 @@ entryPoints: address: ":8443" http: tls: - options: + options: log: level: ERROR providers: - file: + file: filename: /traefik/conf/dynamic.yaml