From f468b3602dad4eeb3aa2306e4bcd10a5d6a38a79 Mon Sep 17 00:00:00 2001 From: Francois Laupretre Date: Thu, 1 Jun 2023 23:49:42 +0000 Subject: [PATCH] A first distribution (AWS secrets manager only) --- .gitignore | 2 + Makefile | 4 + README.md | 236 +++++++++++++++++++++++++++++++++++++++++++++++++++ aws.tf | 30 +++++++ main.tf | 48 +++++++++++ outputs.tf | 23 +++++ variables.tf | 21 +++++ 7 files changed, 364 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 aws.tf create mode 100644 main.tf create mode 100644 outputs.tf create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0537c22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +.idea diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d77d8b6 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ + +doc: + terraform-docs --output-file README.md markdown table . + diff --git a/README.md b/README.md new file mode 100644 index 0000000..d4162d0 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ + +This module receives a string and transforms it by replacing +every secret reference with its actual value, retrieved from the appropriate +central secret store. + +Reference syntax is : + + //@secret/:[:]// + +where : + +- \ is the secret store to search. Today, only 'aws' is + supported. +- \ is the secret name (in the store) +- \ is optional and must be provided when retrieving from a multi-valued secret. On AWS + Secrets Manager, it is optional but common practice. + +Today, only AWS secrets manager is supported but others, like Hashicorp Vault, should be quite easy to add. + +Note that every secret reference must be surrounded by space characters or line start/end. It means that, +if the reference is not at the beginning of a line, it must be preceded with a space character and, if +it is not at the end of a line, it must be followed by a space character. + +## Use case - May I help you ? + +Let's consider a very classical Terraform/Terragrunt configuration +where a git-managed file tree defines a platform, each component associated with a unique subdirectory. +For instance, everything related to the 'dev' environment of the 'myapp' application may be stored +in the 'myapp/dev' directory. + +Let's say that this component/instance is configurable and its configuration parameters are contained +in a YAML file stored in the component directory. Its contents is read by terraform using code like +(terragrunt syntax) : + + locals { + ... + cfg = file("./app_values.yaml") + } + + #---------- + + inputs = { + ... + cfg = local.cfg + } + +Here is an excerpt of the original configuration file : + + ... + newrelic: + license: "eu01xxf2e877c4b8997F24893ac47abNRAL" + collector: "collector.eu01.nr-data.net" + + sentry: + dsn: "https://1654891230161811352@465457211646516o.ingest.sentry.io/465457211646516" + ... + +We see that it exposes some sensitive data. This data is stored in the git repository (and history) +and that's what we want to avoid. + +I initially thought it was a job for +[sealed secrets](https://github.com/bitnami-labs/sealed-secrets), but I often need to +add/modify sensitive values and, after a few days, the manual process to refresh sealed secrets +became too heavy and error-prone. In theory, it could be automated but it is not so easy to do it +in a secure way, and I rapidly gave up. Another important point is that, in their current form, +sealed secrets require manually replicating sensitive data everytime it is +modified and such human-managed synchronization is something I try to eliminate from my environment, +not to add. + +So, I was looking for some sort of 'templating' system that would take a string as input +and replace every occurence of a known pattern with the appropriate +secret value retrieved from various secret stores. I also wanted the system to be flexible : +adding a new secret value to the configuration must not require more than creating the secret +in the secret store and adding a reference to the configuration file. +As I didn't find such a tool, I decided to write a terraform module for it. +And, as I am a nice guy, I am sharing it with you. + +So, let's have a look to our new configuration file : + + ... + newrelic: + license: @secret/aws:cfg-myapp-newrelic:license@ + collector: "collector.eu01.nr-data.net" + + sentry: + dsn: @secret/aws:cfg-myapp-sentry:dsn@ + ... + +You can see that sensitive values have disappeared, replaced by references delimited by '@' characters. +Each reference uniquely identifies the location where the secret value is actually stored. + +About flexibility, note that we just replace what we want. In our example, 'newrelic.collector' is +not considered sensitive and remains stored in clear. This also allows a smoother migration, +creating and replacing secrets at your own pace. + +You may also note that, each time a terraform plan is run, data is retrieved from its original +location, eliminating the manually-managed synchronization process you need when storing values +in clear or using sealed secrets. This makes mechanisms like automated secret rotation +much simpler to implement. + +Now, adding a sensitive value becomes trivial : just create the secret in your secret store and +add a '//...//' reference where it will be replaced by its actual value. + +Here is an example of some terraform code to run the expansion : + + module cfg { + source = "git::git@github.com:flaupretre/terraform-secret-bridge.git?ref=v1.0.0" + + cfg = var.cfg + } + +In this case, the expanded string will be available as 'module.cfg.result'. + +Note that this mechanism is especially well adapted to increase the security level of +an existing project because, when used with a structured document (yaml, json, etc), the document +structure remains unchanged, minimizing the required changes +in the code. It is especially well-suited to stop exposing sensitive data in +a Helm chart 'Values' map (when running Helm through Terraform with a 'helm_release' resource). +There, you just send the yamldecode()d expanded configuration to the chart instead of the original +map and you keep your chart code unchanged. + +## Some notes about security + +Please note that the present process allows to avoid exposing sensitive data in the terraform +repository, but not in the terraform state. Getting rid of sensitive data in the +terraform state or obfuscating it is a much more complex issue and out of scope here. + +So, keep in mind that the 'result' output contains every secret values in clear and +is stored in the terraform state. This is your +responsibility to protect your terraform state, local or remote, from unauthorized access. + +## Secret stores + +### AWS Secrets Manager + +This is the only store we are supporting today. + +AWS secrets generally contain several key/value pairs. Actually, these pairs form a map, +stored as a JSON string. You can decide not to use multiple keys in your secret. In this case, don't +set the ':key' part in your reference and you will retrieve your 'PlainText' secret string as-is. + +## Using a prefix + +You may set an optional prefix string when calling the module. This prefix is added at the beginning +of every secret name before retrieving it from the secret stores. + +This may allow to restrict the set of secrets that may potentially be used in a configuration. +A prefix like 'cfg-myapp-' may be a good way to ensure that my application can +only access a reserved set of secrets defined for itself. + +Using this prefix, the module call we saw before would look like : + + module cfg { + source = "git::git@github.com:flaupretre/terraform-secret-bridge.git?ref=v1.0.0" + + prefix = "cfg-myapp-" + cfg = var.cfg + } + +and the configuration would become : + + ... + newrelic: + license: //aws-secret:newrelic:license// + collector: "collector.eu01.nr-data.net" + + sentry: + dsn: //aws-secret:sentry:dsn// + ... + +## Errors + +When your input string contains a reference to a non-existing secret, execution fails +and you get a message saying that the secret does not exist. + +When the secrets exists but the key you're referencing does not, you receive a message saying : + + │ Error: Invalid function argument + │ + │ on .terraform/modules/cfg/main.tf line 39, in locals: + │ 39: file("***ERROR: '${k}' secret key not found ***") : null + │ + │ Invalid value for "path" parameter: no file exists at "***ERROR: + │ 'aws:sonarcloud:taken' secret key not found (prefix = 'cfg-myapp-') ***"; this function works + │ only with files that are distributed as part of the configuration source + │ code, so if this file will be created by a resource in this configuration + │ you must instead obtain this result from an attribute of that resource. + ╵ + ERRO[0020] 1 error occurred: + * exit status 1 + +Here, the interesting part is 'aws:hd-sonarcloud:tioken' secret key not found' as it gives +the names of the secret store, the secret and the non-existing key. + +From the message above, we can determine that : + +- we are considering secrets stored in AWS Secrets Manager +- The secret name is 'cfg-myapp-sonarcloud' (apply the prefix) +- we try to get a key named 'taken' and it seems it does not exist + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_secretsmanager_secret.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret) | data source | +| [aws_secretsmanager_secret_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret_version) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [input](#input\_input) | A string to expand (contains secret references) | `string` | n/a | yes | +| [prefix](#input\_prefix) | A string to add at the beginning of every secret names | `string` | `""` | no | +| [stores](#input\_stores) | A list of secret stores to consider | `list(string)` | 
[
  "aws"
]
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [result](#output\_result) | The expanded string | + diff --git a/aws.tf b/aws.tf new file mode 100644 index 0000000..72a9eea --- /dev/null +++ b/aws.tf @@ -0,0 +1,30 @@ +# Retrieves secrets from AWS secrets manager + +data aws_secretsmanager_secret this { + for_each = (local.aws_is_up ? { for name in local.names.aws : name => null } : {}) + + name = each.key +} + +#---- + +data aws_secretsmanager_secret_version this { + for_each = (local.aws_is_up ? { for name in local.names.aws : name => null } : {}) + + secret_id = data.aws_secretsmanager_secret.this[each.key].id +} + +#---- + +locals { + + aws_is_up = contains(var.stores, "aws") + + aws_map = (local.aws_is_up ? { + for nk, secret in local.name_key_map.aws : "aws:${nk}" => + (secret.has_key ? + lookup(jsondecode(data.aws_secretsmanager_secret_version.this[secret.name].secret_string) + , secret.key, null) + : data.aws_secretsmanager_secret_version.this[secret.name].secret_string) + } : {}) +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..644b800 --- /dev/null +++ b/main.tf @@ -0,0 +1,48 @@ +# Format is : //aws-secret::// +# + +locals { + + name_key = { for store in var.stores : + store => distinct(flatten(regexall("//@secret/${store}:([^ ]+)//", var.input))) + } + + name_key_map = { for store in var.stores : + store => { + for nk in local.name_key[store] : nk => { + name = "${var.prefix}${element(split(":", nk), 0)}" + has_key = (length(split(":", nk)) > 1) + key = (length(split(":", nk)) > 1) ? element(split(":", nk), 1) : null + } + } + } + + names = { + for store in var.stores : + store => distinct([ + for item in local.name_key_map[store] : item.name + ]) + } + + merged_maps = merge( + local.aws_map, + # When implemented, additional stores will come here + ) + + secret_map = { for k,v in local.merged_maps : + ">//@secret/${k}//" => ((v == null) ? "" : ">\"${v}\"") } + + # This throws an error on secret key not existing + + errors = [ + for k,v in local.merged_maps : (v == null) ? + file("***ERROR: '${k}' secret key not found (prefix = '${var.prefix}') ***") : null + ] + + words = split(" ", replace(replace(">${var.input}", " ", " >"), "\n", " \n >")) + + ewords = [ for word in local.words : lookup(local.secret_map, word, word) ] + + result = substr(replace(replace(join(" ", local.ewords), " \n >", "\n"), " >", " "), 1, 100000) + +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..21d26f2 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,23 @@ + +output result { + description = "The expanded string" + value = local.result + sensitive = true +} + +# For sebugging only + +output words { + value = local.words + sensitive = true +} + +output ewords { + value = local.ewords + sensitive = true +} + +output secret_map { + value = local.secret_map + sensitive = true +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..e426629 --- /dev/null +++ b/variables.tf @@ -0,0 +1,21 @@ + +# Default: activate every supported stores + +variable stores { + description = "A list of secret stores to consider" + type = list(string) + default = [ + "aws" + ] +} + +variable prefix { + description = "A string to add at the beginning of every secret names" + type = string + default = "" +} + +variable input { + description = "A string to expand (contains secret references)" + type = string +}