diff --git a/Makefile b/Makefile index f2fc52c..136cbc4 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,8 @@ BUILDKITE_PIPELINE_SLUG=my_pipeline DUMP_ENV ?= true VAULT_DEV_ROOT_TOKEN_ID ?= 88F4384B-98E9-4AE3-B00C-F55678F89080 -TESTER_VAULT_VERSION ?= 0.9.1 -SVC_VAULT_VERSION ?= 0.9.1 +TESTER_VAULT_VERSION ?= 1.11.2 +SVC_VAULT_VERSION ?= 1.11.2 all:;: '$(VAULT_ADDR)' \ '$(TESTER_VAULT_VERSION)' \ diff --git a/README.md b/README.md index b741c46..8ddb902 100755 --- a/README.md +++ b/README.md @@ -13,20 +13,26 @@ Different types of secrets are supported and exposed to your builds in appropria - Environment Variables for strings - `git-credential` via git's credential.helper -## Example +## ENV example -The following pipeline downloads a private key from `https://my-vault-server/data/buildkite/{pipeline}/ssh_private_key` and set of environment variables from `https://my-vault-server/data/buildkite/{pipeline}/environment`. +The following pipeline downloads env secrets stored in `https://my-vault-server/secret/buildkite/{pipeline}/env` and git-credentials from `https://my-vault-server/secret/buildkite/{pipeline}/git-credentials` -The private key is exposed to both the checkout and the command as an ssh-agent instance. The secrets in the env file are exposed as environment variables. +The keys in the `env` secret are exposed in the `checkout` and `command` as environment variables. The git-credentials are exposed as an environment variable `GIT_CONFIG_PARAMETERS` and are also exposed in the `checkout` and `command`. ```yml steps: - command: ./run_build.sh plugins: - - mikeknox/vault-secrets#v0.1.: - server: my-vault-server + - buildkite-plugins/vault-secrets#v0.2.0: + server: "https://my-vault-server" + path: secret/buildkite + auth: + method: approle + role-id: "my-role-id" + secret-env: "VAULT_SECRET_ID" ``` + ## Uploading Secrets Secrets are uploading using the Vault CLI, as a `base64` encoded blob in a field called *value*. diff --git a/hooks/environment b/hooks/environment index 41702dd..47ff35b 100755 --- a/hooks/environment +++ b/hooks/environment @@ -24,17 +24,13 @@ processSshSecrets() { processEnvSecrets() { local vaultServer="$1" - local envKey="$2" + local key="$2" local envscript='' - - echo "Checking vault key ${envKey}" >&2 - for key in $(list_secrets "$vaultServer" "$envKey") ; do - echo "Downloading env secret from ${key}" >&2; + echo "Downloading env secret from ${key}" >&2; if ! envscript=$(echo "${envscript:-}" && secret_download "${vaultServer}" "${key}") ; then echo "+++ :warning: Failed to download env from $key" >&2; exit 1 - fi - done + fi echo "Evaluating ${#envscript} bytes of env" set -o allexport eval "$envscript" @@ -69,7 +65,7 @@ _source="${BASH_SOURCE[0]}" [ -z "${_source:-}" ] && _source="${0}" basedir="$( cd "$( dirname "${_source}" )" && cd .. && pwd )" -# shellcheck disable=SC1090 +# shellcheck source=lib/shared.bash . "$basedir/lib/shared.bash" TMPDIR=${TMPDIR:-/tmp} @@ -90,7 +86,6 @@ if [[ -n "$vault_server" ]] ; then "${vault_path}" "${VAULT_BASE_PATH}" ) - secrets='' for key in ${secret_paths[*]} ; do echo "Checking vault secrets ${key}" >&2 @@ -101,7 +96,6 @@ if [[ -n "$vault_server" ]] ; then # shellcheck disable=SC2034 _key_base="$(dirname "$secret")" _key_name="$(basename "$secret")" - case "${_key_name:-}" in env|environment) processEnvSecrets "$vault_server" "$secret" ;; private_ssh_key|id_rsa_github) processSshSecrets "$vault_server" "$secret" ;; diff --git a/lib/shared.bash b/lib/shared.bash index 55ee3cd..07704af 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -11,25 +11,40 @@ esac vault_auth() { local server="$1" - local auth_params='' - # role is defined in Vault Configuration - # -header= # used to set X-Auth - # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD - aws - # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER - # BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE - [ -n "${server:-}" ] && auth_params="${auth_params} -address=${server}" - [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" ] && auth_params="${auth_params} -method=${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD}" - [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER:-}" ] && auth_params="${auth_params} -header_value=${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER}" - [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE:-}" ] && auth_params="${auth_params} role=${BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE}" - - if [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" ] ; then - # don't output the token to log, even though it's a temporary token - # shellcheck disable=SC2086 - vault auth $auth_params | grep -v ^token: + + # Currently we only support AppRole authentication. + # These values are referenced when authenticating to the Vault server: + # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD - approle + + # RoleID and SecretID should be stored securely on the agent, there are probably better ways to do this, but here is a start + # We'll use these two values for the RoleID and SecretID: + # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_SECRET_ID + # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID + # + # For now, you will need to define the secret ID oustide of the plugin, though this will probably change. + + # approle authentication + if [ "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" = "approle" ]; then + + secret_var="${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_SECRET_ENV:-$VAULT_SECRET_ID}" + + if [[ -z "${!secret_var:-}" ]]; then + echo "+++ 🚨 No vault secret id found in \$${secret_var}" + exit 1 + fi + + # export the vault token to be used for this job - this command writes to the auth/approle/login endpoint + # on success, vault will return the token which we export as VAULT_TOKEN for this shell + export VAULT_TOKEN + if ! VAULT_TOKEN=$(vault write -field=token -address="$server" auth/approle/login \ + role_id="$BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID" \ + secret_id="${!secret_var:-}"); then + echo "Failed to get vault token" + fi + + echo "Successfully authenticated with RoleID ${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID} and updated vault token" + return "${PIPESTATUS[0]}" - else - # shellcheck disable=SC2086 - echo "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_TOKEN:-}" | vault auth ${auth_params:-} - fi } @@ -38,7 +53,11 @@ list_secrets() { local key="$2" local _list - _list=$(vault list -address="$server" -format=yaml "$key" | sed 's/^- //g' ) + + if ! _list=$(vault kv list -address="$server" -format=yaml "$key" | sed 's/^- //g'); then + echo "unable to list secrets" >&2; + return "${PIPESTATUS[0]}" + fi local retVal=${PIPESTATUS[0]} for lineItem in ${_list} ; do @@ -57,7 +76,7 @@ secret_exists() { local _key_name _key_name="$(basename "$key")" local _list - _list=$(vault list -address="$server" -format=yaml "$_key_base" ) + _list=$(vault kv list -address="$server" -format=yaml "$_key_base" ) echo "${_list}" | grep "^- ${_key_name}$" >& /dev/null # shellcheck disable=SC2181 @@ -71,8 +90,10 @@ secret_exists() { secret_download() { local server="$1" local key="$2" - - _secret=$(vault read -address="${server}" -field=value "$key" | base64 $BASE64_DECODE_ARGS) + if ! _secret=$(vault kv get -field=data -format=yaml secret/buildkite/env | sed 's/: /=/g' ); then + echo "Failed to download secrets" + exit 1 + fi # shellcheck disable=SC2181 if [ "$?" -ne 0 ] ; then return 1 @@ -89,6 +110,7 @@ add_ssh_private_key_to_agent() { fi echo "Loading ssh-key into ssh-agent (pid ${SSH_AGENT_PID:-})" >&2; + echo "$ssh_key" | env SSH_ASKPASS="/bin/false" ssh-add - } diff --git a/plugin.yml b/plugin.yml index 1c4086e..847fcc2 100644 --- a/plugin.yml +++ b/plugin.yml @@ -1,6 +1,6 @@ name: "Vault Secrets" description: "Expose build secrets stored in Vault to your jobs" -author: "@mikeknox" +author: "@mikeknox, @buildkite-plugins" public: true requirements: - curl @@ -10,7 +10,19 @@ configuration: properties: server: type: string + auth: + type: object + properties: + method: + enum: + - 'approle' + - '' + role-id: + type: string required: - server + - auth additionalProperties: false + dependencies: + auth: [ method ]