diff --git a/fixtures/bad_pack/README.md b/fixtures/bad_pack/README.md deleted file mode 100644 index 5bf718ac..00000000 --- a/fixtures/bad_pack/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Simple Service - -This pack is a used to deploy a Docker image to as a service job to Nomad. - -This is ideal for configuring and deploying a simple web application to Nomad. - -## Customizing the Docker Image - -The docker image deployed can be replaced with a variable. In the example -below, we will deploy and run `httpd:latest`. - -``` -nomad-pack run simple_service --var image="httpd:latest" -``` - -## Customizing Ports - -The ports that are exposed via Docker can be customized as well. - -In this case, we'll write the port values to a file called `./overrides.hcl`: - -``` -{ - name = "http" - port = 8000 -}, -{ - name = "https" - port = 8001 -} -``` - -Then pass the file into the run command: - -``` -nomad-pack run simple_service -f ./overrides.hcl`" -``` - -## Customizing Resources - -The application resource limits can be customized: - -``` -resources = { - cpu = 500 - memory = 501 -} -``` - -## Customizing Environment Variables - -Environment variables can be added: - -``` -env_vars = [ - { - key = "foo" - value = 1 - } -] -``` - -## Consul Service and Load Balancer Integration - -Optionally, this pack can configure a Consul service. - -If the `register_consul_service` is unset or set to true, the Consul service will be registered. - -Several load balancers in the [Nomad Pack Community Registry](../README.md) are configured to connect to -this service with ease. - -The [NginX](../nginx/README.md) and [HAProxy](../haproxy/README.md) packs can be configured to balance over the -Consul service deployed by this pack. Just ensure that the "consul_service_name" variable provided to those -packs matches this consul_service_name. - -The [Fabio](../fabio/README.md) and [Traefik](../traefik/README.md) packs are configured to search for Consul -services with the specific tags. - -To tag this Consul service to work with Fabio, add `"urlprefix-"` -to the consul_tags. For instance, to route at the root path, you would add `"urlprefix-/"`. To route at the path `"/api/v1"`, you would add '"urlprefix-/api/v1". - -To tag this Consul service to work with Traefik, add "traefik.enable=true" to the consul_tags, also add "traefik.http.routers.http.rule=Path(\`\`)". To route at the root path, you would add "traefik.http.routers.http.rule=Path(\`/\`)". To route at the path "/api/v1", you would add "traefik.http.routers.http.rule=Path(\`/api/v1\`)". - -``` -register_consul_service = true - -consul_tags = [ - "urlprefix-/", - "traefik.enable=true", - "traefik.http.routers.http.rule=Path(`/`)", -] -``` - -## Customizing Consul and Upstream Services - -Consul configuration can be tweaked and (upstream services)[https://www.nomadproject.io/docs/job-specification/upstreams] -can be added as well. - -``` -register_consul_service = true -consul_service_name = "app-service-name" -has_health_check = true -health_check = { - path = "/health" - interval = "20s" - timeout = "3s" -} -upstreams = [ - { - name = "other-service" - port = 8001 - } -] -``` diff --git a/fixtures/bad_pack/metadata.hcl b/fixtures/bad_pack/metadata.hcl deleted file mode 100644 index 91dd9a40..00000000 --- a/fixtures/bad_pack/metadata.hcl +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -app { - url = "https://learn.hashicorp.com/tutorials/nomad/get-started-run?in=nomad/get-started" -} - -pack { - name = "simple_service" - description = "This deploys a simple service job to Nomad that runs a docker container." - version = "0.0.1" -} diff --git a/fixtures/bad_pack/outputs.tpl b/fixtures/bad_pack/outputs.tpl deleted file mode 100644 index fadfce77..00000000 --- a/fixtures/bad_pack/outputs.tpl +++ /dev/null @@ -1,14 +0,0 @@ -You deployed a service to Nomad. - -There are [[ .simple_service.count ]] instances of your job now running. - -The service is using the image: [[.simple_service.image | quote]] - -[[ if .simple_service.register_consul_service ]] -You registered an associated Consul service named [[ .simple_service.consul_service_name ]]. - -[[ if .simple_service.has_health_check ]] -This service has a health check at the path : [[ .simple_service.health_check.path | quote ]] -[[ end ]] -[[ end ]] - diff --git a/fixtures/bad_pack/templates/_helpers.tpl b/fixtures/bad_pack/templates/_helpers.tpl deleted file mode 100644 index 4565d5bd..00000000 --- a/fixtures/bad_pack/templates/_helpers.tpl +++ /dev/null @@ -1,17 +0,0 @@ -// allow nomad-pack to set the job name - -[[- define "job_name" -]] -[[- if eq .simple_service.job_name "" -]] -[[- .nomad_pack.pack.name | quote -]] -[[- else -]] -[[- .simple_service.job_name | quote -]] -[[- end -]] -[[- end -]] - -// only deploys to a region if specified - -[[- define "region" -]] -[[- if not (eq .simple_service.region "") -]] -region = [[ .simple_service.region | quote]] -[[- end -]] -[[- end -]] diff --git a/fixtures/bad_pack/templates/simple_service.nomad.tpl b/fixtures/bad_pack/templates/simple_service.nomad.tpl deleted file mode 100644 index 723913a3..00000000 --- a/fixtures/bad_pack/templates/simple_service.nomad.tpl +++ /dev/null @@ -1,81 +0,0 @@ -job [[ template "job_name" . ]] { - [[ template "region" . ]] - datacenters = [[ .simple_service.datacenters | toPrettyJson ]] - type = "service" - - # parse error - config {} - - group "app" { - count = [[ .simple_service.count ]] - - network { - [[- range $port := .simple_service.ports ]] - port [[ $port.name | quote ]] { - to = [[ $port.port ]] - } - [[- end ]] - } - - [[- if .simple_service.register_consul_service ]] - service { - name = "[[ .simple_service.consul_service_name ]]" - port = "[[ .simple_service.consul_service_port ]]" - tags = [[ .simple_service.consul_tags | toPrettyJson ]] - - connect { - sidecar_service { - proxy { - [[- range $upstream := .simple_service.upstreams ]] - upstreams { - destination_name = [[ $upstream.name | quote ]] - local_bind_port = [[ $upstream.port ]] - } - [[- end ]] - } - } - } - - [[- if .simple_service.has_health_check ]] - check { - name = "alive" - type = "http" - path = [[ .simple_service.health_check.path | quote ]] - interval = [[ .simple_service.health_check.interval | quote ]] - timeout = [[ .simple_service.health_check.timeout | quote ]] - } - [[- end ]] - } - [[- end ]] - - restart { - attempts = [[ .simple_service.restart_attempts ]] - interval = "30m" - delay = "15s" - mode = "fail" - } - - task "server" { - driver = "docker" - - config { - image = [[.simple_service.image | quote]] - ports = ["http"] - } - - [[- $env_vars_length := len .simple_service.env_vars ]] - [[- if not (eq $env_vars_length 0) ]] - env { - [[- range $var := .simple_service.env_vars ]] - [[ $var.key ]] = [[ $var.value ]] - [[- end ]] - } - [[- end ]] - - resources { - cpu = [[ .simple_service.resources.cpu ]] - memory = [[ .simple_service.resources.memory ]] - } - } - } -} diff --git a/fixtures/bad_pack/variables.hcl b/fixtures/bad_pack/variables.hcl deleted file mode 100644 index 0a8d3903..00000000 --- a/fixtures/bad_pack/variables.hcl +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - -variable "job_name" { - description = "The name to use as the job name which overrides using the pack name" - type = string - // If "", the pack name will be used - default = "" -} - -variable "region" { - description = "The region where jobs will be deployed" - type = string - default = "" -} - -variable "datacenters" { - description = "A list of datacenters in the region which are eligible for task placement" - type = list(string) - default = ["dc1"] -} - -variable "image" { - description = "" - type = string - default = "mnomitch/hello_world_server" -} - -variable "count" { - description = "The number of app instances to deploy" - type = number - default = 1 -} - -variable "restart_attempts" { - description = "The number of times the task should restart on updates" - type = number - default = 2 -} - -variable "has_health_check" { - description = "If you want to register a health check in consul" - type = bool - default = false -} - -variable "health_check" { - description = "" - type = object({ - path = string - interval = string - timeout = string - }) - - default = { - path = "/" - interval = "10s" - timeout = "2s" - } -} - -variable "upstreams" { - description = "" - type = list(object({ - name = string - port = string - })) -} - -variable "register_consul_service" { - description = "If you want to register a consul service for the job" - type = bool - default = false -} - -variable "ports" { - description = "" - type = list(object({ - name = string - port = number - })) - - default = [{ - name = "http" - port = 8000 - }] -} - -variable "env_vars" { - description = "" - type = list(object({ - key = string - value = string - })) - default = [] -} - -variable "consul_service_name" { - description = "The consul service name for the application" - type = string - default = "service" -} - -variable "consul_service_port" { - description = "The consul service name for the application" - type = string - default = "http" -} - -variable "consul_tags" { - description = "" - type = list(string) - default = [] -} - -variable "resources" { - description = "The resource to assign to the Nginx system task that runs on every client" - type = object({ - cpu = number - memory = number - }) - default = { - cpu = 200, - memory = 256 - } -} - -variable "consul_tags" { - description = "The consul service name for the hello-world application" - type = list(string) - default = [] -} diff --git a/fixtures/connect.nomad b/fixtures/jobspecs/connect.nomad similarity index 100% rename from fixtures/connect.nomad rename to fixtures/jobspecs/connect.nomad diff --git a/fixtures/example-with-meta.nomad b/fixtures/jobspecs/example-with-meta.nomad similarity index 100% rename from fixtures/example-with-meta.nomad rename to fixtures/jobspecs/example-with-meta.nomad diff --git a/fixtures/example.nomad b/fixtures/jobspecs/example.nomad similarity index 100% rename from fixtures/example.nomad rename to fixtures/jobspecs/example.nomad diff --git a/fixtures/simple.nomad b/fixtures/jobspecs/simple.nomad similarity index 100% rename from fixtures/simple.nomad rename to fixtures/jobspecs/simple.nomad diff --git a/fixtures/v1/simple_raw_exec_v1/README.md b/fixtures/v1/simple_raw_exec_v1/README.md new file mode 100644 index 00000000..f508fb4a --- /dev/null +++ b/fixtures/v1/simple_raw_exec_v1/README.md @@ -0,0 +1,4 @@ +# Fixture: `simple_raw_exec_v1` + +This pack is cloned in the Loader tests and broken in various ways to test +error handling diff --git a/fixtures/test_registry/packs/simple_raw_exec/metadata.hcl b/fixtures/v1/simple_raw_exec_v1/metadata.hcl similarity index 100% rename from fixtures/test_registry/packs/simple_raw_exec/metadata.hcl rename to fixtures/v1/simple_raw_exec_v1/metadata.hcl diff --git a/fixtures/test_registry/packs/simple_raw_exec/outputs.tpl b/fixtures/v1/simple_raw_exec_v1/outputs.tpl similarity index 100% rename from fixtures/test_registry/packs/simple_raw_exec/outputs.tpl rename to fixtures/v1/simple_raw_exec_v1/outputs.tpl diff --git a/fixtures/v1/simple_raw_exec_v1/templates/simple_raw_exec.nomad.tpl b/fixtures/v1/simple_raw_exec_v1/templates/simple_raw_exec.nomad.tpl new file mode 100644 index 00000000..01236197 --- /dev/null +++ b/fixtures/v1/simple_raw_exec_v1/templates/simple_raw_exec.nomad.tpl @@ -0,0 +1,29 @@ +job [[ coalesce .simple_raw_exec.job_name .nomad_pack.pack.name ]] { + [[- if empty .simple_raw_exec.region | not ]] + region = [[quote .simple_raw_exec.region ]] + [[- end ]] + [[- if empty .simple_raw_exec.namespace | not ]] + namespace = [[quote .simple_raw_exec.namespace ]] + [[- end ]] + datacenters = [[ .simple_raw_exec.datacenters | toJson ]] + type = "service" + + group "app" { + count = [[ .simple_raw_exec.count ]] + + task "server" { + driver = "raw_exec" + + config { + command = "/bin/bash" + args = ["-c",[[ quote .simple_raw_exec.command ]]] + } + [[- if (not (empty .simple_raw_exec.env) ) ]] + [[- print "\n\n env {\n" -]] + [[- range $k, $v := .simple_raw_exec.env -]] + [[- printf " %s = %q\n" $k $v -]] + [[- end -]] + [[- print " }" -]][[- end -]][[- print "" ]] + } + } +} diff --git a/fixtures/test_registry/packs/simple_raw_exec/variables.hcl b/fixtures/v1/simple_raw_exec_v1/variables.hcl similarity index 100% rename from fixtures/test_registry/packs/simple_raw_exec/variables.hcl rename to fixtures/v1/simple_raw_exec_v1/variables.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child1/metadata.hcl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child1/metadata.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child1/metadata.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child1/metadata.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child1/variables.hcl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child1/variables.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child1/variables.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child1/variables.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child2/metadata.hcl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child2/metadata.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child2/metadata.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child2/metadata.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl diff --git a/fixtures/test_registry/packs/my_alias_test/deps/child2/variables.hcl b/fixtures/v1/test_registry/packs/my_alias_test/deps/child2/variables.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/deps/child2/variables.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/deps/child2/variables.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/metadata.hcl b/fixtures/v1/test_registry/packs/my_alias_test/metadata.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/metadata.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/metadata.hcl diff --git a/fixtures/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl b/fixtures/v1/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl rename to fixtures/v1/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl diff --git a/fixtures/test_registry/packs/my_alias_test/variables.hcl b/fixtures/v1/test_registry/packs/my_alias_test/variables.hcl similarity index 100% rename from fixtures/test_registry/packs/my_alias_test/variables.hcl rename to fixtures/v1/test_registry/packs/my_alias_test/variables.hcl diff --git a/fixtures/test_registry/packs/simple_docker/CHANGELOG.md b/fixtures/v1/test_registry/packs/simple_docker/CHANGELOG.md similarity index 100% rename from fixtures/test_registry/packs/simple_docker/CHANGELOG.md rename to fixtures/v1/test_registry/packs/simple_docker/CHANGELOG.md diff --git a/fixtures/test_registry/packs/simple_docker/README.md b/fixtures/v1/test_registry/packs/simple_docker/README.md similarity index 100% rename from fixtures/test_registry/packs/simple_docker/README.md rename to fixtures/v1/test_registry/packs/simple_docker/README.md diff --git a/fixtures/test_registry/packs/simple_docker/metadata.hcl b/fixtures/v1/test_registry/packs/simple_docker/metadata.hcl similarity index 100% rename from fixtures/test_registry/packs/simple_docker/metadata.hcl rename to fixtures/v1/test_registry/packs/simple_docker/metadata.hcl diff --git a/fixtures/test_registry/packs/simple_docker/outputs.tpl b/fixtures/v1/test_registry/packs/simple_docker/outputs.tpl similarity index 100% rename from fixtures/test_registry/packs/simple_docker/outputs.tpl rename to fixtures/v1/test_registry/packs/simple_docker/outputs.tpl diff --git a/fixtures/test_registry/packs/simple_docker/templates/_helpers.tpl b/fixtures/v1/test_registry/packs/simple_docker/templates/_helpers.tpl similarity index 100% rename from fixtures/test_registry/packs/simple_docker/templates/_helpers.tpl rename to fixtures/v1/test_registry/packs/simple_docker/templates/_helpers.tpl diff --git a/fixtures/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl b/fixtures/v1/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl similarity index 100% rename from fixtures/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl rename to fixtures/v1/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl diff --git a/fixtures/test_registry/packs/simple_docker/variables.hcl b/fixtures/v1/test_registry/packs/simple_docker/variables.hcl similarity index 100% rename from fixtures/test_registry/packs/simple_docker/variables.hcl rename to fixtures/v1/test_registry/packs/simple_docker/variables.hcl diff --git a/fixtures/test_registry/packs/simple_raw_exec/README.md b/fixtures/v1/test_registry/packs/simple_raw_exec/README.md similarity index 100% rename from fixtures/test_registry/packs/simple_raw_exec/README.md rename to fixtures/v1/test_registry/packs/simple_raw_exec/README.md diff --git a/fixtures/v1/test_registry/packs/simple_raw_exec/metadata.hcl b/fixtures/v1/test_registry/packs/simple_raw_exec/metadata.hcl new file mode 100644 index 00000000..eecf56a5 --- /dev/null +++ b/fixtures/v1/test_registry/packs/simple_raw_exec/metadata.hcl @@ -0,0 +1,14 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" + author = "Nomad Team" # author field deprecated, left here to make sure we don't panic and fail gracefully +} + +pack { + name = "simple_raw_exec" + description = "This is a test fixture pack used because all platforms support raw_exec" + url = "github.com/hashicorp/nomad-pack/fixtures/test_registry/packs/simple-raw-exec" # url field deprecated, left here to make sure we don't panic and fail gracefully + version = "0.0.1" +} diff --git a/fixtures/v1/test_registry/packs/simple_raw_exec/outputs.tpl b/fixtures/v1/test_registry/packs/simple_raw_exec/outputs.tpl new file mode 100644 index 00000000..e69de29b diff --git a/fixtures/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl b/fixtures/v1/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl similarity index 100% rename from fixtures/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl rename to fixtures/v1/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl diff --git a/fixtures/v1/test_registry/packs/simple_raw_exec/variables.hcl b/fixtures/v1/test_registry/packs/simple_raw_exec/variables.hcl new file mode 100644 index 00000000..f5292d38 --- /dev/null +++ b/fixtures/v1/test_registry/packs/simple_raw_exec/variables.hcl @@ -0,0 +1,44 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + default = "" // If "", the pack name will be used +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 1 +} + +variable "command" { + type = string + description = "bash command to run" + default = "echo \"$(date) - Started.\"; while true; do sleep 300; echo -n .; done" +} + +variable "env" { + type = map(string) + description = "environment variable collection" + default = {} +} + +variable "namespace" { + type = string + description = "namespace to run the job in" + default = "" +} \ No newline at end of file diff --git a/fixtures/variable_test/README.md b/fixtures/v1/variable_test/README.md similarity index 99% rename from fixtures/variable_test/README.md rename to fixtures/v1/variable_test/README.md index 00bcf72f..a2b23e59 100644 --- a/fixtures/variable_test/README.md +++ b/fixtures/v1/variable_test/README.md @@ -1,4 +1,3 @@ # Test Fixture - Single Variable Pack This pack can be used to test variable overrides, heredocs, etc. - diff --git a/fixtures/variable_test/heredoc.vars.hcl b/fixtures/v1/variable_test/heredoc.vars.hcl similarity index 100% rename from fixtures/variable_test/heredoc.vars.hcl rename to fixtures/v1/variable_test/heredoc.vars.hcl diff --git a/fixtures/variable_test/input.vars.hcl b/fixtures/v1/variable_test/input.vars.hcl similarity index 100% rename from fixtures/variable_test/input.vars.hcl rename to fixtures/v1/variable_test/input.vars.hcl diff --git a/fixtures/variable_test/unexpected.vars.hcl b/fixtures/v1/variable_test/unexpected.vars.hcl similarity index 100% rename from fixtures/variable_test/unexpected.vars.hcl rename to fixtures/v1/variable_test/unexpected.vars.hcl diff --git a/fixtures/variable_test/variable_test/README.md b/fixtures/v1/variable_test/variable_test/README.md similarity index 100% rename from fixtures/variable_test/variable_test/README.md rename to fixtures/v1/variable_test/variable_test/README.md diff --git a/fixtures/variable_test/variable_test/metadata.hcl b/fixtures/v1/variable_test/variable_test/metadata.hcl similarity index 100% rename from fixtures/variable_test/variable_test/metadata.hcl rename to fixtures/v1/variable_test/variable_test/metadata.hcl diff --git a/fixtures/variable_test/variable_test/templates/test.nomad.tpl b/fixtures/v1/variable_test/variable_test/templates/test.nomad.tpl similarity index 100% rename from fixtures/variable_test/variable_test/templates/test.nomad.tpl rename to fixtures/v1/variable_test/variable_test/templates/test.nomad.tpl diff --git a/fixtures/variable_test/variable_test/variables.hcl b/fixtures/v1/variable_test/variable_test/variables.hcl similarity index 100% rename from fixtures/variable_test/variable_test/variables.hcl rename to fixtures/v1/variable_test/variable_test/variables.hcl diff --git a/fixtures/v2/override_files/deps_test_1/test1.hcl b/fixtures/v2/override_files/deps_test_1/test1.hcl new file mode 100644 index 00000000..3da7016f --- /dev/null +++ b/fixtures/v2/override_files/deps_test_1/test1.hcl @@ -0,0 +1,5 @@ +deps_test_1.job_name = "A" +deps_test_1.child1.job_name = "A.1" +deps_test_1.child1.gc.job_name = "A.1.a" +deps_test_1.child2.job_name = "A.2" +deps_test_1.child2.gc.job_name = "A.2.a" diff --git a/fixtures/v2/override_files/simple_raw_exec/test_01.hcl b/fixtures/v2/override_files/simple_raw_exec/test_01.hcl new file mode 100644 index 00000000..26c5eeda --- /dev/null +++ b/fixtures/v2/override_files/simple_raw_exec/test_01.hcl @@ -0,0 +1 @@ +simple_raw_exec.job_name = "sre" diff --git a/fixtures/v2/override_files/simple_raw_exec/test_02.hcl b/fixtures/v2/override_files/simple_raw_exec/test_02.hcl new file mode 100644 index 00000000..25c36e29 --- /dev/null +++ b/fixtures/v2/override_files/simple_raw_exec/test_02.hcl @@ -0,0 +1,6 @@ +simple_raw_exec.env = { + "NOMAD_TOKEN" = "some awesome token" + "NOMAD_ADDR" = "http://127.0.0.1:4646" +} + +simple_raw_exec.job_name = "sre" diff --git a/fixtures/v2/simple_raw_exec_v2/README.md b/fixtures/v2/simple_raw_exec_v2/README.md new file mode 100644 index 00000000..c0166f28 --- /dev/null +++ b/fixtures/v2/simple_raw_exec_v2/README.md @@ -0,0 +1,4 @@ +# Fixture: `simple_raw_exec_v2` + +This pack is cloned in the Loader tests and broken in various ways to test +error handling diff --git a/fixtures/v2/simple_raw_exec_v2/metadata.hcl b/fixtures/v2/simple_raw_exec_v2/metadata.hcl new file mode 100644 index 00000000..eecf56a5 --- /dev/null +++ b/fixtures/v2/simple_raw_exec_v2/metadata.hcl @@ -0,0 +1,14 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" + author = "Nomad Team" # author field deprecated, left here to make sure we don't panic and fail gracefully +} + +pack { + name = "simple_raw_exec" + description = "This is a test fixture pack used because all platforms support raw_exec" + url = "github.com/hashicorp/nomad-pack/fixtures/test_registry/packs/simple-raw-exec" # url field deprecated, left here to make sure we don't panic and fail gracefully + version = "0.0.1" +} diff --git a/fixtures/v2/simple_raw_exec_v2/templates/simple_raw_exec.nomad.tpl b/fixtures/v2/simple_raw_exec_v2/templates/simple_raw_exec.nomad.tpl new file mode 100644 index 00000000..27b955ce --- /dev/null +++ b/fixtures/v2/simple_raw_exec_v2/templates/simple_raw_exec.nomad.tpl @@ -0,0 +1,29 @@ +job [[ coalesce ( var "job_name" .) (meta "pack.name" .) | quote ]] { + [[- if (var "region" .) ]] + region = [[.region ]] + [[- end ]] + [[- if (var "namespace" .) ]] + namespace = [[ var "namespace" . | quote ]] + [[- end ]] + datacenters = [[ var "datacenters" . | toJson ]] + type = "service" + + group "app" { + count = [[ var "count" . ]] + + task "server" { + driver = "raw_exec" + + config { + command = "/bin/bash" + args = ["-c", [[ var "command" . | quote ]]] + } + [[- if (var "env" .) ]] + [[- print "\n\n env {\n" -]] + [[- range $k, $v := var "env" . -]] + [[- printf " %s = %q\n" $k $v -]] + [[- end -]] + [[- print " }" -]][[- end -]][[- print "" ]] + } + } +} diff --git a/fixtures/v2/simple_raw_exec_v2/variables.hcl b/fixtures/v2/simple_raw_exec_v2/variables.hcl new file mode 100644 index 00000000..f5292d38 --- /dev/null +++ b/fixtures/v2/simple_raw_exec_v2/variables.hcl @@ -0,0 +1,44 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + default = "" // If "", the pack name will be used +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 1 +} + +variable "command" { + type = string + description = "bash command to run" + default = "echo \"$(date) - Started.\"; while true; do sleep 300; echo -n .; done" +} + +variable "env" { + type = map(string) + description = "environment variable collection" + default = {} +} + +variable "namespace" { + type = string + description = "namespace to run the job in" + default = "" +} \ No newline at end of file diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/metadata.hcl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/metadata.hcl new file mode 100644 index 00000000..8cce0cbf --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/metadata.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} + +pack { + name = "grandchild" + description = "render-only child dependency" + version = "0.0.1" +} diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/templates/grandchild.txt.tpl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/templates/grandchild.txt.tpl new file mode 100644 index 00000000..70e5ea00 --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/templates/grandchild.txt.tpl @@ -0,0 +1,3 @@ +"Grandchild" dependency pack's template output + job_name: [[- var "job_name" . ]] +If all is well, this will not be "grandchild" diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/variables.hcl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/variables.hcl new file mode 100644 index 00000000..40ac77b6 --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/deps/grandchild/variables.hcl @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "job name" + type = string + default = "grandchild" +} diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/metadata.hcl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/metadata.hcl new file mode 100644 index 00000000..653880df --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/metadata.hcl @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} + +pack { + name = "child" + description = "render-only child dependency" + version = "0.0.1" +} + +dependency "grandchild" { + alias = "gc" +} diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/templates/child.txt.tpl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/templates/child.txt.tpl new file mode 100644 index 00000000..cde6610d --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/templates/child.txt.tpl @@ -0,0 +1,4 @@ +"Child" dependency pack's template output + job_name: [[- var "job_name" . ]] + gc.job_name: [[- var "job_name" .gc ]] +If all is well, this will not be "child" diff --git a/fixtures/v2/test_registry/packs/deps_test_1/deps/child/variables.hcl b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/variables.hcl new file mode 100644 index 00000000..ad45f071 --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/deps/child/variables.hcl @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "job name" + type = string + default = "child" +} diff --git a/fixtures/v2/test_registry/packs/deps_test_1/metadata.hcl b/fixtures/v2/test_registry/packs/deps_test_1/metadata.hcl new file mode 100644 index 00000000..7d4fe26a --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/metadata.hcl @@ -0,0 +1,21 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + author = "Nomad Team" + url = "" +} + +pack { + name = "deps_test_1" + description = "This pack tests repeated dependencies" + version = "0.0.1" +} + +dependency "child" { + alias = "child1" +} + +dependency "child" { + alias = "child2" +} diff --git a/fixtures/v2/test_registry/packs/deps_test_1/templates/deps_test.txt.tpl b/fixtures/v2/test_registry/packs/deps_test_1/templates/deps_test.txt.tpl new file mode 100644 index 00000000..f5d775ee --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/templates/deps_test.txt.tpl @@ -0,0 +1,5 @@ + job_name: [[- var "job_name" . ]] + child1.job_name: [[- var "job_name" .child1 ]] + gc.child1.job_name: [[- var "job_name" .child1.gc ]] + child2.job_name: [[- var "job_name" .child2 ]] + gc.child2.job_name: [[- var "job_name" .child2.gc ]] diff --git a/fixtures/v2/test_registry/packs/deps_test_1/variables.hcl b/fixtures/v2/test_registry/packs/deps_test_1/variables.hcl new file mode 100644 index 00000000..95150b2a --- /dev/null +++ b/fixtures/v2/test_registry/packs/deps_test_1/variables.hcl @@ -0,0 +1,50 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + default = "deps_test" +} + +variable "test_name" { + description = "This variable allows for configurable test output" + type = string + default = "test" +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 1 +} + +variable "command" { + type = string + description = "bash command to run" + default = "echo \"$(date) - Started.\"; while true; do sleep 300; echo -n .; done" +} + +variable "env" { + type = map(string) + description = "environment variable collection" + default = { "FOO" = "bar", "BAZ" = "QUX" } +} + +variable "test_name" { + type = string + description = "behavior modifying constant" + default = "" +} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/metadata.hcl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/metadata.hcl new file mode 100644 index 00000000..98df8da2 --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/metadata.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} + +pack { + name = "child1" + description = "render-only child dependency" + version = "0.0.1" +} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl new file mode 100644 index 00000000..4010f67e --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/templates/child1.nomad.tpl @@ -0,0 +1 @@ +[[- var "job_name" . -]] diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/variables.hcl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/variables.hcl new file mode 100644 index 00000000..c0a1fdf0 --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child1/variables.hcl @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "job name" + type = string + default = "child1" +} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/metadata.hcl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/metadata.hcl new file mode 100644 index 00000000..c56be01e --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/metadata.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} + +pack { + name = "child2" + description = "render-only child dependency" + version = "0.0.1" +} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl new file mode 100644 index 00000000..4010f67e --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/templates/child2.nomad.tpl @@ -0,0 +1 @@ +[[- var "job_name" . -]] diff --git a/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/variables.hcl b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/variables.hcl new file mode 100644 index 00000000..bcb8da18 --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/deps/child2/variables.hcl @@ -0,0 +1,21 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "job name" + type = string + default = "child2" +} + +variable "complex" { + description = "complex object for rendering" + type = object({ + name = string + address = string + ids = list(string) + lookup = map(object({ + a = number + b = string + })) + }) +} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/metadata.hcl b/fixtures/v2/test_registry/packs/my_alias_test/metadata.hcl new file mode 100644 index 00000000..4c38f236 --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/metadata.hcl @@ -0,0 +1,17 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" + author = "Nomad Team" +} + +pack { + name = "deps_test" + description = "This pack tests dependencies" + url = "github.com/hashicorp/nomad-pack/fixtures/test_registry/packs/deps_test" + version = "0.0.1" +} + +dependency "child1" {} +dependency "child2" {} diff --git a/fixtures/v2/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl b/fixtures/v2/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl new file mode 100644 index 00000000..4010f67e --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/templates/deps_test.nomad.tpl @@ -0,0 +1 @@ +[[- var "job_name" . -]] diff --git a/fixtures/v2/test_registry/packs/my_alias_test/variables.hcl b/fixtures/v2/test_registry/packs/my_alias_test/variables.hcl new file mode 100644 index 00000000..95150b2a --- /dev/null +++ b/fixtures/v2/test_registry/packs/my_alias_test/variables.hcl @@ -0,0 +1,50 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + default = "deps_test" +} + +variable "test_name" { + description = "This variable allows for configurable test output" + type = string + default = "test" +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 1 +} + +variable "command" { + type = string + description = "bash command to run" + default = "echo \"$(date) - Started.\"; while true; do sleep 300; echo -n .; done" +} + +variable "env" { + type = map(string) + description = "environment variable collection" + default = { "FOO" = "bar", "BAZ" = "QUX" } +} + +variable "test_name" { + type = string + description = "behavior modifying constant" + default = "" +} diff --git a/fixtures/v2/test_registry/packs/simple_docker/CHANGELOG.md b/fixtures/v2/test_registry/packs/simple_docker/CHANGELOG.md new file mode 100644 index 00000000..2db922c4 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/CHANGELOG.md @@ -0,0 +1,3 @@ +## Version v0.0.1 (Unreleased) + +Initial Release diff --git a/fixtures/v2/test_registry/packs/simple_docker/README.md b/fixtures/v2/test_registry/packs/simple_docker/README.md new file mode 100644 index 00000000..994aad7f --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/README.md @@ -0,0 +1,64 @@ +# simple_docker + + + +This pack is a simple Nomad job that runs as a service and can be accessed via +HTTP. + +## Pack Usage + + + +### Changing the Message + +To change the message this server responds with, change the "message" variable +when running the pack. + +``` +nomad-pack run simple_docker --var message="Hola Mundo!" +``` + +This tells Nomad Pack to tweak the `MESSAGE` environment variable that the +service reads from. + +### Consul Service and Load Balancer Integration + +Optionally, it can configure a Consul service. + +If the `register_consul_service` is unset or set to true, the Consul service +will be registered. + +Several load balancers in the [Nomad Pack Community Registry][pack-registry] +are configured to connect to this service by default. + +The [NGINX][pack-nginx] and [HAProxy][pack-haproxy] packs are configured to +balance the Consul service `simple_docker-service`, which is the default value +for the `consul_service_name` variable. + +The [Fabio][pack-fabio] and [Traefik][pack-traefik] packs are configured to +search for Consul services with the tags found in the default value of the +`consul_service_tags` variable. + +## Variables + + + +- `message` (string) - The message your application will respond with +- `count` (number) - The number of app instances to deploy +- `job_name` (string) - The name to use as the job name which overrides using + the pack name +- `datacenters` (list of strings) - A list of datacenters in the region which + are eligible for task placement +- `region` (string) - The region where jobs will be deployed +- `register_consul_service` (bool) - If you want to register a consul service + for the job +- `consul_service_tags` (list of string) - The consul service name for the + simple_docker application +- `consul_service_name` (string) - The consul service name for the simple_docker + application + +[pack-registry]: https://github.com/hashicorp/nomad-pack-community-registry +[pack-nginx]: https://github.com/hashicorp/nomad-pack-community-registry/tree/main/packs/nginx/README.md +[pack-haproxy]: https://github.com/hashicorp/nomad-pack-community-registry/tree/main/packs/haproxy/README.md +[pack-fabio]: https://github.com/hashicorp/nomad-pack-community-registry/tree/main/packs/fabio/README.md +[pack-traefik]: https://github.com/hashicorp/nomad-pack-community-registry/tree/main/packs/traefik/traefik/README.md diff --git a/fixtures/v2/test_registry/packs/simple_docker/metadata.hcl b/fixtures/v2/test_registry/packs/simple_docker/metadata.hcl new file mode 100644 index 00000000..97a4e19d --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/metadata.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} +pack { + name = "simple_docker" + description = "" + url = "" + version = "" +} diff --git a/fixtures/v2/test_registry/packs/simple_docker/outputs.tpl b/fixtures/v2/test_registry/packs/simple_docker/outputs.tpl new file mode 100644 index 00000000..8f8de317 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/outputs.tpl @@ -0,0 +1 @@ +Congrats! You deployed the simple_docker pack on Nomad. diff --git a/fixtures/v2/test_registry/packs/simple_docker/templates/_helpers.tpl b/fixtures/v2/test_registry/packs/simple_docker/templates/_helpers.tpl new file mode 100644 index 00000000..8f758b99 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/templates/_helpers.tpl @@ -0,0 +1,98 @@ +// allow nomad-pack to set the job name +[[ define "job_name" ]] +[[- if eq .my.job_name "" -]] +[[- .nomad_pack.pack.name | quote -]] +[[- else -]] +[[- .my.job_name | quote -]] +[[- end ]] +[[- end ]] + +// only deploys to a region if specified +[[ define "region" -]] +[[- if not (eq .my.region "") -]] + region = [[ .my.region | quote]] +[[- end -]] +[[- end -]] + +// Generic constraint +[[ define "constraints" -]] +[[ range $idx, $constraint := . ]] + constraint { + attribute = [[ $constraint.attribute | quote ]] + [[ if $constraint.operator -]] + operator = [[ $constraint.operator | quote ]] + [[ end -]] + value = [[ $constraint.value | quote ]] + } +[[ end -]] +[[- end -]] + +// Generic "service" block template +[[ define "service" -]] +[[ $service := . ]] + service { + name = [[ $service.service_name | quote ]] + port = [[ $service.service_port_label | quote ]] + tags = [[ $service.service_tags | toStringList ]] + [[- if gt (len $service.upstreams) 0 ]] + connect { + sidecar_service { + proxy { + [[- if gt (len $service.upstreams) 0 ]] + [[- range $upstream := $service.upstreams ]] + upstreams { + destination_name = [[ $upstream.name | quote ]] + local_bind_port = [[ $upstream.port ]] + } + [[- end ]] + [[- end ]] + } + } + } + [[- end ]] + check { + type = [[ $service.check_type | quote ]] + [[- if $service.check_path]] + path = [[ $service.check_path | quote ]] + [[- end]] + interval = [[ $service.check_interval | quote ]] + timeout = [[ $service.check_timeout | quote ]] + } + } +[[- end ]] + +// Generic env_vars template +[[ define "env_vars" -]] + [[- range $idx, $var := . ]] + [[ $var.key ]] = [[ $var.value | quote ]] + [[- end ]] +[[- end ]] + + +// Generic mount template +[[ define "mounts" -]] +[[- range $idx, $mount := . ]] + mount { + type = [[ $mount.type | quote ]] + target = [[ $mount.target | quote ]] + source = [[ $mount.source | quote ]] + readonly = [[ $mount.readonly ]] + [[- if gt (len $mount.bind_options) 0 ]] + bind_options { + [[- range $idx, $opt := $mount.bind_options ]] + [[ $opt.name ]] = [[ $opt.value | quote ]] + [[- end ]] + } + [[- end ]] + } +[[- end ]] +[[- end ]] + +// Generic resources template +[[ define "resources" -]] +[[- $resources := . ]] + resources { + cpu = [[ $resources.cpu ]] + memory = [[ $resources.memory ]] + } +[[- end ]] diff --git a/fixtures/v2/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl b/fixtures/v2/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl new file mode 100644 index 00000000..9d7e8348 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/templates/simple_docker.nomad.tpl @@ -0,0 +1,50 @@ +job [[ template "job_name" . ]] { + [[ template "region" . ]] + datacenters = [[ .my.datacenters | toStringList ]] + type = "service" + + group "app" { + count = [[ .my.count ]] + + network { + port "http" { + to = 8000 + } + } + + [[ if .my.register_consul_service ]] + service { + name = "[[ .my.consul_service_name ]]" + tags = [[ .my.consul_service_tags | toStringList ]] + port = "http" + check { + name = "alive" + type = "http" + path = "/" + interval = "10s" + timeout = "2s" + } + } + [[ end ]] + + restart { + attempts = 2 + interval = "30m" + delay = "15s" + mode = "fail" + } + + task "server" { + driver = "docker" + + config { + image = "mnomitch/hello_world_server" + ports = ["http"] + } + + env { + MESSAGE = [[.my.message | quote]] + } + } + } +} diff --git a/fixtures/v2/test_registry/packs/simple_docker/variables.hcl b/fixtures/v2/test_registry/packs/simple_docker/variables.hcl new file mode 100644 index 00000000..aa23e059 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_docker/variables.hcl @@ -0,0 +1,60 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + // If "", the pack name will be used + default = "" +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 2 +} + +variable "message" { + description = "The message your application will render" + type = string + default = "Hello World!" +} + +variable "register_consul_service" { + description = "If you want to register a consul service for the job" + type = bool + default = true +} + +variable "consul_service_name" { + description = "The consul service name for the simple_docker application" + type = string + default = "webapp" +} + +variable "consul_service_tags" { + description = "The consul service name for the simple_docker application" + type = list(string) + // defaults to integrate with Fabio or Traefik + // This routes at the root path "/", to route to this service from + // another path, change "urlprefix-/" to "urlprefix-/" and + // "traefik.http.routers.http.rule=Path(∫/∫)" to + // "traefik.http.routers.http.rule=Path(∫/∫)" + default = [ + "urlprefix-/", + "traefik.enable=true", + "traefik.http.routers.http.rule=Path(`/`)", + ] +} diff --git a/fixtures/v2/test_registry/packs/simple_raw_exec/metadata.hcl b/fixtures/v2/test_registry/packs/simple_raw_exec/metadata.hcl new file mode 100644 index 00000000..eecf56a5 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_raw_exec/metadata.hcl @@ -0,0 +1,14 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" + author = "Nomad Team" # author field deprecated, left here to make sure we don't panic and fail gracefully +} + +pack { + name = "simple_raw_exec" + description = "This is a test fixture pack used because all platforms support raw_exec" + url = "github.com/hashicorp/nomad-pack/fixtures/test_registry/packs/simple-raw-exec" # url field deprecated, left here to make sure we don't panic and fail gracefully + version = "0.0.1" +} diff --git a/fixtures/v2/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl b/fixtures/v2/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl new file mode 100644 index 00000000..70e1365d --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_raw_exec/templates/simple_raw_exec.nomad.tpl @@ -0,0 +1,36 @@ +job [[ coalesce ( var "job_name" .) (meta "pack.name" .) | quote ]] { + [[- if (var "region" .) ]] + region = [[.region ]] + [[- end ]] + [[- if (var "namespace" .) ]] + namespace = [[ var "namespace" . | quote ]] + [[- end ]] + datacenters = [[ var "datacenters" . | toJson ]] + type = "service" + + group "app" { + count = [[ var "count" . ]] + + restart { + attempts = 2 + interval = "30m" + delay = "15s" + mode = "fail" + } + + task "server" { + driver = "raw_exec" + + config { + command = "/bin/bash" + args = ["-c",[[ var "command" . | quote ]]] + } + [[- if (var "env" .) ]] + [[- print "\n\n env {\n" -]] + [[- range $k, $v := var "env" . -]] + [[- printf " %s = %q\n" $k $v -]] + [[- end -]] + [[- print " }" -]][[- end -]][[- print "" ]] + } + } +} diff --git a/fixtures/v2/test_registry/packs/simple_raw_exec/variables.hcl b/fixtures/v2/test_registry/packs/simple_raw_exec/variables.hcl new file mode 100644 index 00000000..d03fcf09 --- /dev/null +++ b/fixtures/v2/test_registry/packs/simple_raw_exec/variables.hcl @@ -0,0 +1,44 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "job_name" { + description = "The name to use as the job name which overrides using the pack name" + type = string + default = "" // If "", the pack name will be used +} + +variable "region" { + description = "The region where jobs will be deployed" + type = string + default = "" +} + +variable "datacenters" { + description = "A list of datacenters in the region which are eligible for task placement" + type = list(string) + default = ["dc1"] +} + +variable "count" { + description = "The number of app instances to deploy" + type = number + default = 1 +} + +variable "command" { + type = string + description = "bash command to run" + default = "echo \"$(date) - Started.\"; while true; do sleep 300; echo -n .; done" +} + +variable "env" { + type = map(string) + description = "environment variable collection" + default = {} +} + +variable "namespace" { + type = string + description = "namespace to run the job in" + default = "" +} diff --git a/fixtures/v2/variable_test/README.md b/fixtures/v2/variable_test/README.md new file mode 100644 index 00000000..a2b23e59 --- /dev/null +++ b/fixtures/v2/variable_test/README.md @@ -0,0 +1,3 @@ +# Test Fixture - Single Variable Pack + +This pack can be used to test variable overrides, heredocs, etc. diff --git a/fixtures/v2/variable_test/heredoc.vars.hcl b/fixtures/v2/variable_test/heredoc.vars.hcl new file mode 100644 index 00000000..3a312212 --- /dev/null +++ b/fixtures/v2/variable_test/heredoc.vars.hcl @@ -0,0 +1,11 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# This file INTENTIONALLY does not end with a newline after the EOF +# and is used to test a parsing condition that occurs when the file +# ends immediately after the end-of-heredoc marker +# https://github.com/hashicorp/nomad-pack/pull/191 + +variable_test_pack.input = < **NOTE**: The pack's folder name doesn't match the pack's name intentionally +> to test error context output when running a pack from the filesystem. + +## Inputs + +* **input** [default: `default`] - This is the only variable used in the + template. The template outputs this value. diff --git a/fixtures/v2/variable_test/variable_test/metadata.hcl b/fixtures/v2/variable_test/variable_test/metadata.hcl new file mode 100644 index 00000000..8d0e008e --- /dev/null +++ b/fixtures/v2/variable_test/variable_test/metadata.hcl @@ -0,0 +1,12 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +app { + url = "" +} + +pack { + name = "variable_test_pack" + description = "This pack tests variable overrides" + version = "0.0.1" +} diff --git a/fixtures/v2/variable_test/variable_test/templates/test.nomad.tpl b/fixtures/v2/variable_test/variable_test/templates/test.nomad.tpl new file mode 100644 index 00000000..9fcdc692 --- /dev/null +++ b/fixtures/v2/variable_test/variable_test/templates/test.nomad.tpl @@ -0,0 +1 @@ +[[ var "input" . ]] diff --git a/fixtures/v2/variable_test/variable_test/variables.hcl b/fixtures/v2/variable_test/variable_test/variables.hcl new file mode 100644 index 00000000..d438aa91 --- /dev/null +++ b/fixtures/v2/variable_test/variable_test/variables.hcl @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "input" { + type = string + description = "Test Heredoc" + default = "default" +} diff --git a/go.mod b/go.mod index 5c513ca2..f543f559 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/hashicorp/go-getter v1.7.2 github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/hcl v1.0.1-vault-5 github.com/hashicorp/hcl/v2 v2.17.1-0.20230725002108-58caf00be5aa github.com/hashicorp/nomad v1.5.0-beta.1.0.20230804093607-388198abef93 github.com/hashicorp/nomad/api v0.0.0-20230804093607-388198abef93 @@ -33,7 +32,9 @@ require ( github.com/shoenig/test v0.6.7 github.com/spf13/afero v1.9.5 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 github.com/zclconf/go-cty v1.14.0 + golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 golang.org/x/term v0.10.0 golang.org/x/text v0.11.0 ) @@ -169,6 +170,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect github.com/hashicorp/hil v0.0.0-20210521165536-27a72121fd40 // indirect github.com/hashicorp/logutils v1.0.0 // indirect github.com/hashicorp/mdns v1.0.4 // indirect @@ -245,7 +247,6 @@ require ( github.com/skeema/knownhosts v1.2.0 // indirect github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/stretchr/testify v1.8.4 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect github.com/tj/go-spin v1.1.0 // indirect @@ -265,7 +266,6 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.13.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index c8177ff1..6cbc4647 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -28,6 +28,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/helper" "github.com/hashicorp/nomad-pack/internal/pkg/helper/filesystem" "github.com/hashicorp/nomad-pack/internal/pkg/logging" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" "github.com/hashicorp/nomad-pack/internal/pkg/version" "github.com/hashicorp/nomad-pack/internal/runner/job" "github.com/hashicorp/nomad-pack/internal/testui" @@ -72,34 +73,34 @@ func TestCLI_CreateTestRegistry(t *testing.T) { t.Logf("match %v: %v\n", i, match) } must.RegexMatch(t, regex, out) - must.Eq(t, 0, result.exitCode) + must.Zero(t, result.exitCode) } func TestCLI_Version(t *testing.T) { t.Parallel() // This test doesn't require a Nomad cluster. exitCode := Main([]string{"nomad-pack", "-v"}) - must.Eq(t, 0, exitCode) + must.Zero(t, exitCode) } func TestCLI_JobRun(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) }) } // Confirm that another pack with the same job names but a different deployment name fails func TestCLI_JobRunConflictingDeployment(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) - result := runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack), "--name=with-name"}) + result := runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack), "--name=with-name"}) must.Eq(t, 1, result.exitCode) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), job.ErrExistsInDeployment{JobID: testPack, Deployment: testPack}.Error()) // Confirm that it's still possible to update the existing pack - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) }) } @@ -107,11 +108,11 @@ func TestCLI_JobRunConflictingDeployment(t *testing.T) { func TestCLI_JobRunConflictingNonPackJob(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { // Register non pack job - err := ct.NomadRun(s, getTestNomadJobPath(testPack)) + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) must.NoError(t, err) // Now try to register the pack - result := runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)}) + result := runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)}) must.Eq(t, 1, result.exitCode) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) @@ -123,11 +124,11 @@ func TestCLI_JobRunConflictingNonPackJob(t *testing.T) { func TestCLI_JobRunConflictingJobWithMeta(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { // Register non pack job - err := ct.NomadRun(s, getTestNomadJobPath("simple_raw_exec_with_meta")) + err := ct.NomadRun(s, getTestNomadJobPath(t, "simple_raw_exec_with_meta")) must.NoError(t, err) // Now try to register the pack - result := runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)}) + result := runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)}) must.Eq(t, 1, result.exitCode) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) @@ -146,7 +147,7 @@ func TestCLI_JobRunFails(t *testing.T) { func TestCLI_JobPlan(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - expectGoodPackPlan(t, runTestPackCmd(t, s, []string{"plan", getTestPackPath(testPack)})) + expectGoodPackPlan(t, runTestPackCmd(t, s, []string{"plan", getTestPackPath(t, testPack)})) }) } @@ -180,11 +181,11 @@ func TestCLI_JobPlan_ConflictingDeployment(t *testing.T) { func TestCLI_JobPlan_ConflictingNonPackJob(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { // Register non pack job - err := ct.NomadRun(s, getTestNomadJobPath(testPack)) + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) must.NoError(t, err) // Now try to register the pack - result := runTestPackCmd(t, s, []string{"plan", getTestPackPath(testPack)}) + result := runTestPackCmd(t, s, []string{"plan", getTestPackPath(t, testPack)}) must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) must.Eq(t, 255, result.exitCode) // Should return 255 indicating an error @@ -193,33 +194,37 @@ func TestCLI_JobPlan_ConflictingNonPackJob(t *testing.T) { func TestCLI_PackPlan_OverrideExitCodes(t *testing.T) { ct.HTTPTest(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - // Plan against empty - should be makes-changes - result := runTestPackCmd(t, s, []string{ - "plan", - "--exit-code-makes-changes=91", - "--exit-code-no-changes=90", - "--exit-code-error=92", - getTestPackPath(testPack), - }) - must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) - must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") - must.Eq(t, 91, result.exitCode) // Should return exit-code-makes-changes + t.Run("plan_against_empty", func(t *testing.T) { + // Plan against empty - should be makes-changes + result := runTestPackCmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackPath(t, testPack), + }) + must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) + must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") + must.Eq(t, 91, result.exitCode) // Should return exit-code-makes-changes + }) // Register non pack job - err := ct.NomadRun(s, getTestNomadJobPath(testPack)) + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) must.NoError(t, err) - // Now try to register the pack, should make error - result = runTestPackCmd(t, s, []string{ - "plan", - "--exit-code-makes-changes=91", - "--exit-code-no-changes=90", - "--exit-code-error=92", - getTestPackPath(testPack), + t.Run("register_pack_expect_error", func(t *testing.T) { + // Now try to register the pack, should make error + result := runTestPackCmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackPath(t, testPack), + }) + must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) + must.Eq(t, 92, result.exitCode) // Should exit-code-error }) - must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) - must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) - must.Eq(t, 92, result.exitCode) // Should exit-code-error err = ct.NomadPurge(s, testPack) must.NoError(t, err) @@ -237,10 +242,11 @@ func TestCLI_PackPlan_OverrideExitCodes(t *testing.T) { wait.Gap(500*time.Millisecond), ), must.Sprint("test job failed to purge")) - result = runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)}) + // Make a pack deployment so we can validate the "no-change" condition + result := runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)}) must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), "") - must.Eq(t, 0, result.exitCode) // Should return 0 + must.Zero(t, result.exitCode) // Should return 0 isStarted := func() bool { j, err := ct.NomadJobStatus(s, testPack) if err != nil { @@ -254,28 +260,31 @@ func TestCLI_PackPlan_OverrideExitCodes(t *testing.T) { wait.Gap(500*time.Millisecond), ), must.Sprint("test job failed to start")) - // Plan against deployed - should be no-changes - result = runTestPackCmd(t, s, []string{ - "plan", - "--exit-code-makes-changes=91", - "--exit-code-no-changes=90", - "--exit-code-error=92", - getTestPackPath(testPack), + t.Run("pack_against_deployed", func(t *testing.T) { + + // Plan against deployed - should be no-changes + result = runTestPackCmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackPath(t, testPack), + }) + must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) + must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") + must.Eq(t, 90, result.exitCode, must.Sprintf("stdout:\n%s\n\nstderr:\n%s\n", result.cmdOut.String(), result.cmdErr.String())) // Should return exit-code-no-changes }) - must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) - must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") - must.Eq(t, 90, result.exitCode) // Should return exit-code-no-changes }) } func TestCLI_PackStop(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) - result := runTestPackCmd(t, s, []string{"stop", getTestPackPath(testPack), "--purge=true"}) + result := runTestPackCmd(t, s, []string{"stop", getTestPackPath(t, testPack), "--purge=true"}) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), `Pack "`+testPack+`" destroyed`) - must.Eq(t, 0, result.exitCode) + must.Zero(t, result.exitCode) }) } @@ -327,16 +336,16 @@ func TestCLI_PackStop_Conflicts(t *testing.T) { // Create job if tC.nonPackJob { - err = ct.NomadRun(s, getTestNomadJobPath(testPack)) + err = ct.NomadRun(s, getTestNomadJobPath(t, testPack)) must.NoError(t, err) } else { deploymentName := fmt.Sprintf("--name=%s", tC.deploymentName) - varJobName := fmt.Sprintf("--var=job_name=%s", tC.jobName) + varJobName := fmt.Sprintf("--var=%s.job_name=%s", testPack, tC.jobName) if tC.namespace != "" { namespaceFlag := fmt.Sprintf("--namespace=%s", tC.namespace) - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack), deploymentName, varJobName, namespaceFlag})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack), deploymentName, varJobName, namespaceFlag})) } else { - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack), deploymentName, varJobName})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack), deploymentName, varJobName})) } } @@ -352,11 +361,11 @@ func TestCLI_PackStop_Conflicts(t *testing.T) { // test that specific functionality func TestCLI_PackDestroy(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)})) + expectGoodPackDeploy(t, runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)})) - result := runTestPackCmd(t, s, []string{"destroy", getTestPackPath(testPack)}) + result := runTestPackCmd(t, s, []string{"destroy", getTestPackPath(t, testPack)}) must.StrContains(t, result.cmdOut.String(), `Pack "`+testPack+`" destroyed`) - must.Eq(t, 0, result.exitCode) + must.Zero(t, result.exitCode) // Assert job no longer queryable c, err := ct.NewTestClient(s) @@ -383,17 +392,17 @@ func TestCLI_PackDestroy_WithOverrides(t *testing.T) { jobNames := []string{"foo", "bar"} for _, j := range jobNames { expectGoodPackDeploy(t, runTestPackCmd( - t, s, []string{"run", testPack, "--var=job_name=" + j, "--registry=" + reg.Name})) + t, s, []string{"run", testPack, "--var=" + testPack + ".job_name=" + j, "--registry=" + reg.Name})) } // Stop nonexistent job - result := runTestPackCmd(t, s, []string{"destroy", testPack, "--var=job_name=baz", "--registry=" + reg.Name}) + result := runTestPackCmd(t, s, []string{"destroy", testPack, "--var=" + testPack + ".job_name=baz", "--registry=" + reg.Name}) must.Eq(t, 1, result.exitCode, must.Sprintf( "expected exitcode 1; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) // Stop job with var override - result = runTestPackCmd(t, s, []string{"destroy", testPack, "--var=job_name=foo", "--registry=" + reg.Name}) - must.Eq(t, 0, result.exitCode, must.Sprintf( + result = runTestPackCmd(t, s, []string{"destroy", testPack, "--var=" + testPack + ".job_name=foo", "--registry=" + reg.Name}) + must.Zero(t, result.exitCode, must.Sprintf( "expected exitcode 0; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) // Assert job "bar" still exists @@ -403,7 +412,7 @@ func TestCLI_PackDestroy_WithOverrides(t *testing.T) { // Stop job with no overrides passed result = runTestPackCmd(t, s, []string{"destroy", testPack, "--registry=" + reg.Name}) - must.Eq(t, 0, result.exitCode, must.Sprintf( + must.Zero(t, result.exitCode, must.Sprintf( "expected exitcode 0; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) // Assert job bar is gone @@ -430,8 +439,8 @@ func TestCLI_CLIFlag_NotDefined(t *testing.T) { func TestCLI_PackStatus(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { - result := runTestPackCmd(t, s, []string{"run", getTestPackPath(testPack)}) - must.Eq(t, 0, result.exitCode) + result := runTestPackCmd(t, s, []string{"run", getTestPackPath(t, testPack)}) + must.Zero(t, result.exitCode) testcases := []struct { name string @@ -459,7 +468,7 @@ func TestCLI_PackStatus(t *testing.T) { t.Run(tc.name, func(t *testing.T) { args := append([]string{"status"}, tc.args...) result := runTestPackCmd(t, s, args) - must.Eq(t, 0, result.exitCode) + must.Zero(t, result.exitCode) must.StrContains(t, result.cmdOut.String(), "simple_raw_exec | "+cache.DevRegistryName+" ") }) } @@ -469,11 +478,11 @@ func TestCLI_PackStatus(t *testing.T) { func TestCLI_PackStatus_Fails(t *testing.T) { ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { // test for status on missing pack - result := runTestPackCmd(t, s, []string{"status", getTestPackPath(testPack)}) + result := runTestPackCmd(t, s, []string{"status", getTestPackPath(t, testPack)}) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) - must.StrContains(t, result.cmdOut.String(), "no jobs found for pack \""+getTestPackPath(testPack)+"\"") + must.StrContains(t, result.cmdOut.String(), "no jobs found for pack \""+getTestPackPath(t, testPack)+"\"") // FIXME: Should this have a non-success exit-code? - must.Eq(t, 0, result.exitCode) + must.Zero(t, result.exitCode) // test flag validation for name flag without pack result = runTestPackCmd(t, s, []string{"status", "--name=foo"}) @@ -488,15 +497,15 @@ func TestCLI_PackRender_MyAlias(t *testing.T) { // output is not guaranteed to be ordered. This requires that the test handle // either order. expected := []string{ - "child1/child1.nomad=child1", - "child2/child2.nomad=child2", + "deps_test/child1/child1.nomad=child1", + "deps_test/child2/child2.nomad=child2", "deps_test/deps_test.nomad=deps_test", } result := runPackCmd(t, []string{ "render", "--no-format=true", - getTestPackPath("my_alias_test"), + getTestPackPath(t, "my_alias_test"), }) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) @@ -532,7 +541,7 @@ func TestCLI_CLIFlag_Namespace(t *testing.T) { { desc: "flags vs job", args: []string{ - `--var=namespace=job`, + `--var=` + testPack + `.namespace=job`, `--namespace=flag`, }, env: make(map[string]string), @@ -566,7 +575,7 @@ func TestCLI_CLIFlag_Namespace(t *testing.T) { result := runTestPackCmd(t, srv, append([]string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), }, tC.args...), ) @@ -593,7 +602,7 @@ func TestCLI_CLIFlag_Token(t *testing.T) { result := runTestPackCmd(t, srv, []string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), "--token=bad00000-bad0-bad0-bad0-badbadbadbad", }) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) @@ -602,7 +611,7 @@ func TestCLI_CLIFlag_Token(t *testing.T) { result = runTestPackCmd(t, srv, []string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), "--token=" + srv.Config.Client.Meta["token"], }) @@ -622,7 +631,7 @@ func TestCLI_EnvConfig_Token(t *testing.T) { result := runTestPackCmd(t, srv, []string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), }) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) must.StrContains(t, result.cmdOut.String(), "403 (ACL token not found)", must.Sprintf( @@ -632,7 +641,7 @@ func TestCLI_EnvConfig_Token(t *testing.T) { t.Setenv("NOMAD_TOKEN", srv.Config.Client.Meta["token"]) result = runTestPackCmd(t, srv, []string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), }) must.Eq(t, result.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) @@ -661,7 +670,7 @@ func TestCLI_EnvConfig_Namespace(t *testing.T) { { desc: "env vs job", args: []string{ - `--var=namespace=job`, + `--var=` + testPack + `.namespace=job`, }, expect: map[string]int{ "job": 1, @@ -684,7 +693,7 @@ func TestCLI_EnvConfig_Namespace(t *testing.T) { desc: "env vs flag vs job", args: []string{ `--namespace=flag`, - `--var=namespace=job`, + `--var=` + testPack + `.namespace=job`, }, expect: map[string]int{ "job": 1, @@ -705,7 +714,7 @@ func TestCLI_EnvConfig_Namespace(t *testing.T) { t.Setenv("NOMAD_NAMESPACE", "env") result := runTestPackCmd(t, srv, append([]string{ "run", - getTestPackPath(testPack), + getTestPackPath(t, testPack), }, tC.args...), ) @@ -732,15 +741,18 @@ type PackCommandResult struct { } func AddressFromTestServer(srv *agent.TestAgent) []string { + srv.T.Helper() return []string{"--address", srv.HTTPAddr()} } func runTestPackCmd(t *testing.T, srv *agent.TestAgent, args []string) PackCommandResult { + t.Helper() args = append(args, AddressFromTestServer(srv)...) return runPackCmd(t, args) } func runPackCmd(t *testing.T, args []string) PackCommandResult { + t.Helper() cmdOut := bytes.NewBuffer(make([]byte, 0)) cmdErr := bytes.NewBuffer(make([]byte, 0)) @@ -788,42 +800,41 @@ func runPackCmd(t *testing.T, args []string) PackCommandResult { // Test Helper functions // getTestPackPath returns the full path to a pack in the test fixtures folder. -func getTestPackPath(packname string) string { - return path.Join(getTestPackRegistryPath(), "packs", packname) +func getTestPackPath(t *testing.T, packname string) string { + t.Helper() + return path.Join(getTestPackRegistryPath(t), "packs", packname) } // getTestPackRegistryPath returns the full path to a registry in the test // fixtures folder. -func getTestPackRegistryPath() string { - return path.Join(testFixturePath(), "test_registry") +func getTestPackRegistryPath(t *testing.T) string { + t.Helper() + return path.Join(testfixture.AbsPath(t, "v2/test_registry")) } // getTestNomadJobPath returns the full path to a pack in the test // fixtures/jobspecs folder. The `.nomad` extension will be added // for you. -func getTestNomadJobPath(job string) string { - return path.Join(testFixturePath(), "jobspecs", job+".nomad") -} - -func testFixturePath() string { - // This is a function to prevent a massive refactor if this ever needs to be - // dynamically determined. - return "../../fixtures/" +func getTestNomadJobPath(t *testing.T, job string) string { + t.Helper() + return path.Join(testfixture.AbsPath(t, path.Join("jobspecs", job+".nomad"))) } // expectGoodPackDeploy bundles the test expectations that should be met when // determining if the pack CLI successfully deployed a pack. func expectGoodPackDeploy(t *testing.T, r PackCommandResult) { - must.Eq(t, r.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", r.cmdErr.String())) + t.Helper() + expectNoStdErrOutput(t, r) must.StrContains(t, r.cmdOut.String(), "Pack successfully deployed", must.Sprintf( "Expected success message, received %q", r.cmdOut.String())) - must.Eq(t, 0, r.exitCode) + must.Zero(t, r.exitCode) } // expectGoodPackPlan bundles the test expectations that should be met when // determining if the pack CLI successfully planned a pack. func expectGoodPackPlan(t *testing.T, r PackCommandResult) { - must.Eq(t, r.cmdErr.String(), "", must.Sprintf("cmdErr should be empty, but was %q", r.cmdErr.String())) + t.Helper() + expectNoStdErrOutput(t, r) must.StrContains(t, r.cmdOut.String(), "Plan succeeded", must.Sprintf( "Expected success message, received %q", r.cmdOut.String())) must.Eq(t, 1, r.exitCode) // exitcode 1 means that an allocation will be created @@ -833,6 +844,8 @@ func expectGoodPackPlan(t *testing.T, r PackCommandResult) { // second one has testRef ref. It returns registry objects, and a string that // points to the root where the two refs are on the filesystem. func createTestRegistries(t *testing.T) (*cache.Registry, *cache.Registry, string) { + t.Helper() + // Fake a clone registryName := fmt.Sprintf("test-%v", time.Now().UnixMilli()) @@ -842,7 +855,7 @@ func createTestRegistries(t *testing.T) (*cache.Registry, *cache.Registry, strin for _, r := range []string{"latest", testRef} { must.NoError(t, filesystem.CopyDir( - getTestPackPath(testPack), + getTestPackPath(t, testPack), path.Join(regDir, r, testPack+"@"+r), false, logging.Default(), @@ -874,5 +887,11 @@ func createTestRegistries(t *testing.T) (*cache.Registry, *cache.Registry, strin } func cleanTestRegistry(t *testing.T, regPath string) { + t.Helper() os.RemoveAll(regPath) } + +func expectNoStdErrOutput(t *testing.T, r PackCommandResult) { + t.Helper() + must.Eq(t, "", r.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", r.cmdErr.String())) +} diff --git a/internal/cli/cli_v1_test.go b/internal/cli/cli_v1_test.go new file mode 100644 index 00000000..a212745a --- /dev/null +++ b/internal/cli/cli_v1_test.go @@ -0,0 +1,807 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path" + "strings" + "testing" + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/command/agent" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" + "github.com/shoenig/test/wait" + + ct "github.com/hashicorp/nomad-pack/internal/cli/testhelper" + "github.com/hashicorp/nomad-pack/internal/pkg/cache" + flag "github.com/hashicorp/nomad-pack/internal/pkg/flag" + "github.com/hashicorp/nomad-pack/internal/pkg/helper" + "github.com/hashicorp/nomad-pack/internal/pkg/helper/filesystem" + "github.com/hashicorp/nomad-pack/internal/pkg/logging" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" + "github.com/hashicorp/nomad-pack/internal/pkg/version" + "github.com/hashicorp/nomad-pack/internal/runner/job" + "github.com/hashicorp/nomad-pack/internal/testui" +) + +func TestCLI_V1_JobRun(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)})) + }) +} + +// Confirm that another pack with the same job names but a different deployment name fails +func TestCLI_V1_JobRunConflictingDeployment(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)})) + + result := runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack), "--name=with-name"}) + must.Eq(t, 1, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsInDeployment{JobID: testPack, Deployment: testPack}.Error()) + + // Confirm that it's still possible to update the existing pack + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)})) + }) +} + +// Check for conflict with non-pack job i.e. no meta +func TestCLI_V1_JobRunConflictingNonPackJob(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + // Register non pack job + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) + must.NoError(t, err) + + // Now try to register the pack + result := runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)}) + + must.Eq(t, 1, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) + }) +} + +// Check for conflict with job that has meta +func TestCLI_V1_JobRunConflictingJobWithMeta(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + // Register non pack job + err := ct.NomadRun(s, getTestNomadJobPath(t, "simple_raw_exec_with_meta")) + must.NoError(t, err) + + // Now try to register the pack + result := runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)}) + must.Eq(t, 1, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) + }) +} + +func TestCLI_V1_JobRunFails(t *testing.T) { + t.Parallel() + // This test doesn't require a Nomad cluster. + result := runPackV1Cmd(t, []string{"run", "fake-job"}) + + must.Eq(t, 1, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Failed To Find Pack") +} + +func TestCLI_V1_JobPlan(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + expectGoodPackPlan(t, runTestPackV1Cmd(t, s, []string{"plan", getTestPackV1Path(t, testPack)})) + }) +} + +func TestCLI_V1_JobPlan_BadJob(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + result := runTestPackV1Cmd(t, s, []string{"plan", "fake-job"}) + + must.Eq(t, 255, result.exitCode) // Should return 255 indicating an error + must.Eq(t, "", result.cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", result.cmdErr.String())) + must.StrContains(t, result.cmdOut.String(), "Failed To Find Pack") + }) +} + +// Confirm that another pack with the same job names but a different deployment name fails +func TestCLI_V1_JobPlan_ConflictingDeployment(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + reg, _, regPath := createV1TestRegistries(t) + defer cleanTestRegistry(t, regPath) + + testRegFlag := "--registry=" + reg.Name + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", testPack, testRegFlag})) + + result := runTestPackV1Cmd(t, s, []string{"run", testPack, testRegFlag, testRefFlag}) + must.Eq(t, 1, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsInDeployment{JobID: testPack, Deployment: testPack + "@latest"}.Error()) + }) +} + +// Check for conflict with non-pack job i.e. no meta +func TestCLI_V1_JobPlan_ConflictingNonPackJob(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + // Register non pack job + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) + must.NoError(t, err) + + // Now try to register the pack + result := runTestPackV1Cmd(t, s, []string{"plan", getTestPackV1Path(t, testPack)}) + must.Eq(t, 255, result.exitCode) // Should return 255 indicating an error + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) + }) +} + +func TestCLI_V1_PackPlan_OverrideExitCodes(t *testing.T) { + ct.HTTPTest(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + // Plan against empty - should be makes-changes + result := runTestPackV1Cmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackV1Path(t, testPack), + }) + must.Eq(t, 91, result.exitCode) // Should return exit-code-makes-changes + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") + + // Register non pack job + err := ct.NomadRun(s, getTestNomadJobPath(t, testPack)) + must.NoError(t, err) + + // Now try to register the pack, should make error + result = runTestPackV1Cmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackV1Path(t, testPack), + }) + must.Eq(t, 92, result.exitCode) // Should exit-code-error + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), job.ErrExistsNonPack{JobID: testPack}.Error()) + + err = ct.NomadPurge(s, testPack) + must.NoError(t, err) + + isGone := func() bool { + _, err = ct.NomadJobStatus(s, testPack) + if err != nil { + return err.Error() == "Unexpected response code: 404 (job not found)" + } + return false + } + must.Wait(t, wait.InitialSuccess( + wait.BoolFunc(isGone), + wait.Timeout(10*time.Second), + wait.Gap(500*time.Millisecond), + ), must.Sprint("test job failed to purge")) + + result = runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)}) + must.Zero(t, result.exitCode) // Should return 0 + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "") + + isStarted := func() bool { + j, err := ct.NomadJobStatus(s, testPack) + if err != nil { + return false + } + return *j.Status == "running" + } + must.Wait(t, wait.InitialSuccess( + wait.BoolFunc(isStarted), + wait.Timeout(30*time.Second), + wait.Gap(500*time.Millisecond), + ), must.Sprint("test job failed to start")) + + // Plan against deployed - should be no-changes + result = runTestPackV1Cmd(t, s, []string{ + "plan", + "--exit-code-makes-changes=91", + "--exit-code-no-changes=90", + "--exit-code-error=92", + getTestPackV1Path(t, testPack), + }) + must.Eq(t, 90, result.exitCode) // Should return exit-code-no-changes + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Plan succeeded\n") + }) +} + +func TestCLI_V1_PackStop(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)})) + + result := runTestPackV1Cmd(t, s, []string{"stop", getTestPackV1Path(t, testPack), "--purge=true"}) + must.Zero(t, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), `Pack "`+testPack+`" destroyed`) + }) +} + +func TestCLI_V1_PackStop_Conflicts(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + + testCases := []struct { + desc string + nonPackJob bool + packName string + deploymentName string + jobName string + namespace string + }{ + // Give these each different job names so there's no conflicts + // between the different tests cases when running + { + desc: "non-pack-job", + nonPackJob: true, + packName: testPack, + deploymentName: "", + jobName: testPack, + }, + { + desc: "same-pack-diff-deploy", + nonPackJob: false, + packName: testPack, + deploymentName: "foo", + jobName: "job2", + }, + { + desc: "same-pack-diff-namespace", + nonPackJob: false, + packName: testPack, + deploymentName: "", + jobName: testPack, + namespace: "job", + }, + } + client, err := ct.NewTestClient(s) + must.NoError(t, err) + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + defer ct.NomadCleanup(s) + + if tC.namespace != "" { + ct.MakeTestNamespaces(t, client) + } + + // Create job + if tC.nonPackJob { + err = ct.NomadRun(s, getTestNomadJobPath(t, testPack)) + must.NoError(t, err) + } else { + deploymentName := fmt.Sprintf("--name=%s", tC.deploymentName) + varJobName := fmt.Sprintf("--var=job_name=%s", tC.jobName) + if tC.namespace != "" { + namespaceFlag := fmt.Sprintf("--namespace=%s", tC.namespace) + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack), deploymentName, varJobName, namespaceFlag})) + } else { + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack), deploymentName, varJobName})) + } + } + + // Try to stop job + result := runTestPackV1Cmd(t, s, []string{"stop", tC.packName}) + must.Eq(t, 1, result.exitCode) + }) + } + }) +} + +// Destroy is just an alias for stop --purge so we only need to +// test that specific functionality +func TestCLI_V1_PackDestroy(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + expectGoodPackDeploy(t, runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)})) + + result := runTestPackV1Cmd(t, s, []string{"destroy", getTestPackV1Path(t, testPack)}) + must.Eq(t, 0, result.exitCode) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), `Pack "`+testPack+`" destroyed`) + + // Assert job no longer queryable + c, err := ct.NewTestClient(s) + must.NoError(t, err) + + j, _, err := c.Jobs().Info("bar", &api.QueryOptions{}) + must.Nil(t, j) + must.ErrorContains(t, err, "job not found") + }) +} + +// Test that destroy properly uses var overrides to target the job +func TestCLI_V1_PackDestroy_WithOverrides(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + c, err := ct.NewTestClient(s) + must.NoError(t, err) + // Because this test uses ref, it requires a populated pack cache. + reg, _, regPath := createV1TestRegistries(t) + defer cleanTestRegistry(t, regPath) + + // TODO: Table Testing + // Create multiple jobs in the same pack deployment + + jobNames := []string{"foo", "bar"} + for _, j := range jobNames { + expectGoodPackDeploy(t, runTestPackV1Cmd( + t, + s, + []string{ + "run", + testPack, + "--var=job_name=" + j, + "--registry=" + reg.Name, + })) + } + + // Stop nonexistent job + result := runTestPackV1Cmd(t, s, []string{"destroy", testPack, "--var=job_name=baz", "--registry=" + reg.Name}) + must.Eq(t, 1, result.exitCode, must.Sprintf("expected exitcode 1; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) + + // Stop job with var override + result = runTestPackV1Cmd(t, s, []string{"destroy", testPack, "--var=job_name=foo", "--registry=" + reg.Name}) + must.Zero(t, result.exitCode, must.Sprintf("expected exitcode 0; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) + + q := api.QueryOptions{} + + // Assert job "bar" still exists + tCtx, done := context.WithTimeout(context.TODO(), 5*time.Second) + j, _, err := c.Jobs().Info("bar", q.WithContext(tCtx)) + done() + must.NoError(t, err) + must.NotNil(t, j) + + // Stop job with no overrides passed + result = runTestPackV1Cmd(t, s, []string{"destroy", testPack, "--registry=" + reg.Name}) + must.Zero(t, result.exitCode, must.Sprintf("expected exitcode 0; got %v\ncmdOut:%v", result.exitCode, result.cmdOut.String())) + + // Assert job bar is gone + tCtx, done = context.WithTimeout(context.TODO(), 5*time.Second) + j, _, err = c.Jobs().Info("bar", q.WithContext(tCtx)) + done() + must.Error(t, err) + must.Eq(t, "Unexpected response code: 404 (job not found)", err.Error()) + must.Nil(t, j) + }) +} + +func TestCLI_V1_CLIFlag_NotDefined(t *testing.T) { + t.Parallel() // nomad not required + + // There is no job flag. This tests that adding an unspecified flag does not + // create an invalid memory address error + // Posix case + result := runPackV1Cmd(t, []string{"run", "nginx", "--job=provided-but-not-defined"}) + must.Eq(t, 1, result.exitCode) + + // std go case + result = runPackV1Cmd(t, []string{"run", "-job=provided-but-not-defined", "nginx"}) + must.Eq(t, 1, result.exitCode) +} + +func TestCLI_V1_PackStatus(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + result := runTestPackV1Cmd(t, s, []string{"run", getTestPackV1Path(t, testPack)}) + must.Zero(t, result.exitCode) + + testcases := []struct { + name string + args []string + }{ + { + name: "no-pack-name", + args: []string{}, + }, + { + name: "with-pack-name", + args: []string{testPack}, + }, + { + name: "with-pack-and-registry-name", + args: []string{testPack, "--registry=default"}, + }, + { + name: "with-pack-and-ref", + args: []string{testPack, "--ref=latest"}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"status"}, tc.args...) + result := runTestPackV1Cmd(t, s, args) + must.Zero(t, result.exitCode) + must.StrContains(t, result.cmdOut.String(), "simple_raw_exec | "+cache.DevRegistryName+" ") + }) + } + }) +} + +func TestCLI_V1_PackStatus_Fails(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(s *agent.TestAgent) { + // test for status on missing pack + result := runTestPackV1Cmd(t, s, []string{"status", getTestPackV1Path(t, testPack)}) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "no jobs found for pack \""+getTestPackV1Path(t, testPack)+"\"") + // FIXME: Should this have a non-success exit-code? + must.Zero(t, result.exitCode) + + // test flag validation for name flag without pack + result = runTestPackV1Cmd(t, s, []string{"status", "--name=foo"}) + must.Eq(t, 1, result.exitCode) + must.StrContains(t, result.cmdOut.String(), "--name can only be used if pack name is provided") + }) +} + +func TestCLI_V1_PackRender_MyAlias(t *testing.T) { + t.Parallel() + // This test has to do some extra shenanigans because dependent pack template + // output is not guaranteed to be ordered. This requires that the test handle + // either order. + expected := []string{ + "child1/child1.nomad=child1", + "child2/child2.nomad=child2", + "deps_test/deps_test.nomad=deps_test", + } + + result := runPackV1Cmd(t, []string{ + "render", + "--no-format", + getTestPackV1Path(t, "my_alias_test"), + }) + expectNoStdErrOutput(t, result) + output := result.cmdOut.String() + must.StrNotContains(t, output, "failed to render") + + // Performing a little clever string manipulation on the render output to + // prepare it for splitting into a slice of string enables us to use + // must.CliceContainsAll to validate goodness. + outStr := strings.TrimSpace(output) + outStr = strings.ReplaceAll(outStr, ":\n\n", "=") + elems := strings.Split(outStr, "\n") + + must.SliceContainsAll(t, expected, elems, must.Sprintf("expected: %v\n got: %v", expected, elems)) +} + +func TestCLI_V1_CLIFlag_Namespace(t *testing.T) { + testCases := []struct { + desc string + args []string + env map[string]string + expect map[string]int + }{ + { + desc: "flags vs unspecified", + args: []string{ + `--namespace=flag`, + }, + env: make(map[string]string), + expect: map[string]int{ + "job": 0, + "flag": 1, + "env": 0, + }, + }, + { + desc: "flags vs job", + args: []string{ + `--var=namespace=job`, + `--namespace=flag`, + }, + env: make(map[string]string), + expect: map[string]int{ + "job": 1, + "flag": 0, + "env": 0, + }, + }, + { + desc: "flags vs second flag", + args: []string{ + `--namespace=job`, + `--namespace=flag`, + }, + env: make(map[string]string), + expect: map[string]int{ + "job": 0, + "flag": 1, + "env": 0, + }, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + ct.HTTPTestParallel(t, ct.WithDefaultConfig(), func(srv *agent.TestAgent) { + c, err := ct.NewTestClient(srv) + must.NoError(t, err) + + ct.MakeTestNamespaces(t, c) + + result := runTestPackV1Cmd(t, srv, append([]string{ + "run", + getTestPackV1Path(t, testPack), + }, + tC.args...), + ) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Pack successfully deployed", must.Sprintf("Expected success message, received %q", result.cmdOut.String())) + + for ns, count := range tC.expect { + j, _, err := c.Jobs().List(&api.QueryOptions{Namespace: ns}) + must.NoError(t, err) + must.Eq(t, count, len(j), must.Sprintf( + "Expected %v job(s) in %q namespace; found %v", count, ns, len(j))) + } + }) + }) + } +} + +func TestCLI_V1_CLIFlag_Token(t *testing.T) { + ct.HTTPTestWithACLParallel(t, ct.WithDefaultConfig(), func(srv *agent.TestAgent) { + c, err := ct.NewTestClient(srv) + must.NoError(t, err) + + ct.MakeTestNamespaces(t, c) + + result := runTestPackV1Cmd(t, srv, []string{ + "run", + getTestPackV1Path(t, testPack), + "--token=bad00000-bad0-bad0-bad0-badbadbadbad", + }) + + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Unexpected response code: 403 (ACL token not found)", must.Sprintf("Expected token not found error, received %q", result.cmdOut.String())) + + result = runTestPackV1Cmd(t, srv, []string{ + "run", + getTestPackV1Path(t, testPack), + "--token=" + srv.Config.Client.Meta["token"], + }) + + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Pack successfully deployed", must.Sprintf("Expected success message, received %q", result.cmdOut.String())) + }) +} + +func TestCLI_V1_EnvConfig_Token(t *testing.T) { + ct.HTTPTestWithACL(t, ct.WithDefaultConfig(), func(srv *agent.TestAgent) { + // Garbage token - Should fail + t.Setenv("NOMAD_TOKEN", badACLToken) + + result := runTestPackV1Cmd(t, srv, []string{ + "run", + getTestPackV1Path(t, testPack), + }) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Unexpected response code: 403 (ACL token not found)", must.Sprintf("Expected token not found error, received %q", result.cmdOut.String())) + + // Good token - Should run + t.Setenv("NOMAD_TOKEN", srv.Config.Client.Meta["token"]) + result = runTestPackV1Cmd(t, srv, []string{ + "run", + getTestPackV1Path(t, testPack), + }) + + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Pack successfully deployed", must.Sprintf("Expected success message, received %q", result.cmdOut.String())) + }) +} + +// This test can't benefit from parallelism since it mutates the environment +func TestCLI_V1_EnvConfig_Namespace(t *testing.T) { + testCases := []struct { + desc string + args []string + env map[string]string + expect map[string]int + }{ + { + desc: "flags vs unspecified", + args: []string{}, + expect: map[string]int{ + "job": 0, + "flag": 0, + "env": 1, + }, + }, + { + desc: "env vs job", + args: []string{ + `--var=namespace=job`, + }, + expect: map[string]int{ + "job": 1, + "flag": 0, + "env": 0, + }, + }, + { + desc: "env vs flag", + args: []string{ + `--namespace=flag`, + }, + expect: map[string]int{ + "job": 0, + "flag": 1, + "env": 0, + }, + }, + { + desc: "env vs flag vs job", + args: []string{ + `--namespace=flag`, + `--var=namespace=job`, + }, + expect: map[string]int{ + "job": 1, + "flag": 0, + "env": 0, + }, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + ct.HTTPTest(t, ct.WithDefaultConfig(), func(srv *agent.TestAgent) { + c, err := ct.NewTestClient(srv) + must.NoError(t, err) + + ct.MakeTestNamespaces(t, c) + + // Always set the namespace environment variable + t.Setenv("NOMAD_NAMESPACE", "env") + result := runTestPackV1Cmd(t, srv, append([]string{ + "run", + getTestPackV1Path(t, testPack), + }, + tC.args...), + ) + expectNoStdErrOutput(t, result) + must.StrContains(t, result.cmdOut.String(), "Pack successfully deployed", must.Sprintf("Expected success message, received %q", result.cmdOut.String())) + + for ns, count := range tC.expect { + j, _, err := c.Jobs().List(&api.QueryOptions{Namespace: ns}) + must.NoError(t, err) + must.Eq(t, count, len(j), must.Sprintf( + "Expected %v job(s) in %q namespace; found %v", count, ns, len(j))) + } + }) + }) + } +} + +func TLSConfigFromTestServer(srv *agent.TestAgent) []string { + srv.T.Helper() + if srv.Config.TLSConfig == nil { + return []string{} + } + return []string{ + "--client-cert", srv.Config.TLSConfig.CertFile, + "--client-key", srv.Config.TLSConfig.KeyFile, + "--ca-cert", srv.Config.TLSConfig.CAFile, + } +} + +func runTestPackV1Cmd(t *testing.T, srv *agent.TestAgent, args []string) PackCommandResult { + t.Helper() + args = append(args, AddressFromTestServer(srv)...) + return runPackV1Cmd(t, args) +} + +func runPackV1Cmd(t *testing.T, args []string) PackCommandResult { + t.Helper() + args = append(args, "--parser-v1") + + cmdOut := bytes.NewBuffer(make([]byte, 0)) + cmdErr := bytes.NewBuffer(make([]byte, 0)) + + // Build our cancellation context + ctx, closer := helper.WithInterrupt(context.Background()) + defer closer() + + // Make a test UI + ui := testui.NonInteractiveTestUI(ctx, cmdOut, cmdErr) + + // Get our base command + fset := flag.NewSets() + base, commands := Commands(ctx, WithFlags(fset), WithUI(ui)) + defer base.Close() + + command := &cli.CLI{ + Name: "nomad-pack", + Args: args, + Version: fmt.Sprintf("Nomad Pack %s", version.HumanVersion()), + Commands: commands, + Autocomplete: true, + AutocompleteNoDefaultFlags: true, + HelpFunc: GroupedHelpFunc(cli.BasicHelpFunc(cliName)), + HelpWriter: cmdOut, + ErrorWriter: cmdErr, + } + + t.Logf("Running nomad-pack\n args:%v", command.Args) + + // Run the CLI + exitCode, err := command.Run() + if err != nil { + panic(err) + } + + must.Eq(t, "", cmdErr.String(), must.Sprintf("cmdErr should be empty, but was %q", cmdErr.String())) + + return PackCommandResult{ + exitCode: exitCode, + cmdOut: cmdOut, + cmdErr: cmdErr, + } +} + +// getTestPackPath returns the full path to a pack in the test fixtures folder. +func getTestPackV1Path(t *testing.T, packname string) string { + t.Helper() + return path.Join(getTestPackRegistryV1Path(t), "packs", packname) +} + +// getTestPackRegistryPath returns the full path to a registry in the test +// fixtures folder. +func getTestPackRegistryV1Path(t *testing.T) string { + t.Helper() + return path.Join(testfixture.AbsPath(t, "v1/test_registry")) +} + +// createTestRegistries creates two registries: first one has "latest" ref, +// second one has testRef ref. It returns registry objects, and a string that +// points to the root where the two refs are on the filesystem. +func createV1TestRegistries(t *testing.T) (*cache.Registry, *cache.Registry, string) { + t.Helper() + + // Fake a clone + registryName := fmt.Sprintf("test-%v", time.Now().UnixMilli()) + + regDir := path.Join(cache.DefaultCachePath(), registryName) + err := filesystem.MaybeCreateDestinationDir(regDir) + must.NoError(t, err) + + for _, r := range []string{"latest", testRef} { + must.NoError(t, filesystem.CopyDir( + getTestPackV1Path(t, testPack), + path.Join(regDir, r, testPack+"@"+r), + false, + logging.Default(), + )) + } + + // create output registries and metadata.json files + latestReg := &cache.Registry{ + Name: registryName, + Source: "github.com/hashicorp/nomad-pack-test-registry", + LocalRef: testRef, + Ref: "latest", + } + latestMetaPath := path.Join(regDir, "latest", "metadata.json") + b, _ := json.Marshal(latestReg) + must.NoError(t, os.WriteFile(latestMetaPath, b, 0644)) + + testRefReg := &cache.Registry{ + Name: registryName, + Source: "github.com/hashicorp/nomad-pack-test-registry", + LocalRef: testRef, + Ref: testRef, + } + testRefMetaPath := path.Join(regDir, testRef, "metadata.json") + b, _ = json.Marshal(testRefReg) + must.NoError(t, os.WriteFile(testRefMetaPath, b, 0644)) + + return latestReg, testRefReg, regDir +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 4b3ffd83..367a30a0 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -16,7 +16,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/cache" flag "github.com/hashicorp/nomad-pack/internal/pkg/flag" - "github.com/hashicorp/nomad-pack/internal/pkg/variable" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/envloader" "github.com/hashicorp/nomad-pack/terminal" ) @@ -71,6 +71,9 @@ type baseCommand struct { // one instance of a pack within the same cluster deploymentName string + // useParserV1 is true when the user supplies the --parser-v1 flag + useParserV1 bool + // args that were present after parsing flags args []string @@ -185,7 +188,7 @@ func (c *baseCommand) Init(opts ...Option) error { } c.args = baseCfg.Flags.Args() - c.envVars = variable.GetVarsFromEnv() + c.envVars = envloader.New().GetVarsFromEnv() // Do any validation after parsing if baseCfg.Validation != nil { @@ -274,6 +277,15 @@ func (c *baseCommand) flagSet(bit flagSetBit, f func(*flag.Sets)) *flag.Sets { the name that should be passed to nomad-pack's plan and destroy commands.`, }) + + f.BoolVar(&flag.BoolVar{ + Name: "parser-v1", + Target: &c.useParserV1, + Default: false, + Usage: `Use the legacy syntax parser to parse your job. This + enables pack to run packs for earlier versions while you are + migrating them to the new syntax`, + }) } if bit&flagSetNeedsApproval != 0 { f := set.NewSet("Approval Options") diff --git a/internal/cli/generate_varfile.go b/internal/cli/generate_varfile.go index 332332bd..e88f07c3 100644 --- a/internal/cli/generate_varfile.go +++ b/internal/cli/generate_varfile.go @@ -96,7 +96,7 @@ func (c *generateVarFileCommand) writeFile(path string, content string) error { return err } if !overwrite { - return fmt.Errorf("destination file exists and overwrite is unset") + return errors.New("destination file exists and overwrite is unset") } } diff --git a/internal/cli/helpers.go b/internal/cli/helpers.go index 21f52e1d..ec8bf19f 100644 --- a/internal/cli/helpers.go +++ b/internal/cli/helpers.go @@ -14,7 +14,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/errors" "github.com/hashicorp/nomad-pack/internal/pkg/manager" "github.com/hashicorp/nomad-pack/internal/pkg/renderer" - "github.com/hashicorp/nomad-pack/internal/pkg/variable" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" "github.com/hashicorp/nomad-pack/internal/runner" "github.com/hashicorp/nomad-pack/internal/runner/job" "github.com/hashicorp/nomad-pack/terminal" @@ -43,6 +43,7 @@ func generatePackManager(c *baseCommand, client *api.Client, packCfg *cache.Pack VariableFiles: c.varFiles, VariableCLIArgs: c.vars, VariableEnvVars: c.envVars, + UseParserV1: c.useParserV1, } return manager.NewPackManager(&cfg, client) } @@ -159,7 +160,7 @@ func renderVariableOverrideFile( manager *manager.PackManager, ui terminal.UI, errCtx *errors.UIErrorContext, -) (*variable.ParsedVariables, error) { +) (*parser.ParsedVariables, error) { r, err := manager.ProcessVariableFiles() if err != nil { @@ -216,7 +217,7 @@ func getPackJobsByDeploy(c *api.Client, cfg *cache.PackConfig, deploymentName st return nil, fmt.Errorf("error finding jobs for pack %s: %s", cfg.Name, err) } if len(jobs) == 0 { - return nil, fmt.Errorf("no job(s) found") + return nil, errors.New("no job(s) found") } var packJobs []*api.Job diff --git a/internal/cli/info.go b/internal/cli/info.go index 1c6fb1d6..b44a8e32 100644 --- a/internal/cli/info.go +++ b/internal/cli/info.go @@ -10,7 +10,9 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/cache" "github.com/hashicorp/nomad-pack/internal/pkg/flag" "github.com/hashicorp/nomad-pack/internal/pkg/loader" - "github.com/hashicorp/nomad-pack/internal/pkg/variable" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/sdk/pack" "github.com/mitchellh/go-glint" "github.com/zclconf/go-cty/cty" ) @@ -47,15 +49,15 @@ func (c *InfoCommand) Run(args []string) int { packPath := c.packConfig.Path - pack, err := loader.Load(packPath) + p, err := loader.Load(packPath) if err != nil { c.ui.ErrorWithContext(err, "failed to load pack from local directory", errorContext.GetAll()...) return 1 } - variableParser, err := variable.NewParser(&variable.ParserConfig{ - ParentName: path.Base(packPath), - RootVariableFiles: pack.RootVariableFiles(), + variableParser, err := parser.NewParser(&config.ParserConfig{ + ParentPackID: pack.ID(path.Base(packPath)), + RootVariableFiles: p.RootVariableFiles(), IgnoreMissingVars: c.baseCommand.ignoreMissingVars, }) if err != nil { @@ -73,20 +75,20 @@ func (c *InfoCommand) Run(args []string) int { doc.Append(glint.Layout( glint.Style(glint.Text("Pack Name "), glint.Bold()), - glint.Text(pack.Metadata.Pack.Name), + glint.Text(p.Metadata.Pack.Name), ).Row()) doc.Append(glint.Layout( glint.Style(glint.Text("Description "), glint.Bold()), - glint.Text(pack.Metadata.Pack.Description), + glint.Text(p.Metadata.Pack.Description), ).Row()) doc.Append(glint.Layout( glint.Style(glint.Text("Application URL "), glint.Bold()), - glint.Text(pack.Metadata.App.URL), + glint.Text(p.Metadata.App.URL), ).Row()) - for pName, variables := range parsedVars.Vars { + for pName, variables := range parsedVars.GetVars() { doc.Append(glint.Layout( glint.Style(glint.Text(fmt.Sprintf("Pack %q Variables:", pName)), glint.Bold()), diff --git a/internal/cli/render.go b/internal/cli/render.go index 094c91da..8763dddd 100644 --- a/internal/cli/render.go +++ b/internal/cli/render.go @@ -9,9 +9,11 @@ import ( "io/fs" "os" "path" + "slices" "strings" "github.com/posener/complete" + "golang.org/x/exp/maps" "github.com/hashicorp/nomad-pack/internal/pkg/cache" "github.com/hashicorp/nomad-pack/internal/pkg/errors" @@ -147,7 +149,7 @@ func writeFile(c *RenderCommand, path string, content string) error { return err } if !overwrite { - return fmt.Errorf("destination file exists and overwrite is unset") + return errors.New("destination file exists and overwrite is unset") } } @@ -159,13 +161,88 @@ func writeFile(c *RenderCommand, path string, content string) error { return nil } -// formatRenderName trims the low-value elements from the rendered template -// name. -func formatRenderName(name string) string { - outName := strings.Replace(name, "/templates/", "/", 1) - outName = strings.TrimSuffix(outName, ".tpl") +// rangeRenders populates a slice of `Render` (rendered templates) such that the +// target slice is sorted by Pack ID, Filename. +func rangeRenders(subj map[string]string, target *[]Render) { - return outName + // The problem: the keys don't trivially sort in pack order anymore because + // they have both "/template/" and a filename in them. + + // Declare some types to make the map key types a bit more obvious. + type PackKey string // pack key + type Filename string // filename + type Content string // render + + // The rendered templates are in a map[string]string, with the key being the + // pack-relative path to the template and the value being the rendered + // template's file content. Dependency packs will have more path components + // before the `/templates/` component. + + // Build a map that contains the Template slices produced by the renderer. The + // key of the map is a pack-relative path to the template, with dependency + // packs being child elements of the pack that depends on them. + packKeySet := make(map[PackKey]map[Filename]Content) + for k, v := range subj { + + // Using strings.Cut with `/templates/` provides the pack key in the + // `before` and the template filename in the `after`. This also trims + // `/templates/` out of the produced key as a side-effect since it's + // low value. + key, val, _ := strings.Cut(k, "/templates/") + + var packKey = PackKey(key) + + // Remove the .tpl from the rendered template filenames + var filename = Filename(strings.TrimSuffix(val, ".tpl")) + + // If this is the first time we have encountered this pack's key, + // we need to build the map to hold the Filename and content. + if _, found := packKeySet[packKey]; !found { + packKeySet[packKey] = make(map[Filename]Content) + } + + // Add the template content to the map + packKeySet[packKey][filename] = Content(v) + } + + // At this point, we have a map[PackKey]map[Filename]Content. Sorting the + // outer map's keys, accessing that element, and then sorting the inner + // map's keys (Filename), enables us to rewrite the target []Render in + // Pack, Filename order should be able to to some sorting and traversing into + // an ordered slice. + + // Grab a list of the pack keys and sort them. Note, they are full pack- + // relative paths, so they nicely sort in depth-sensitive way + packKeys := maps.Keys(packKeySet) + slices.Sort(packKeys) + + // Range the sorted list of pack keys... + for _, packKey := range packKeys { + + // Grab the map[Filename]Content + mFileContent := packKeySet[packKey] + + // Extract the keys as a slice + filenames := maps.Keys(mFileContent) + + // Sort the filenames alphabetically + slices.Sort(filenames) + + // Range the sorted list of filenames... + for _, filename := range filenames { + // Grab the Content + content := mFileContent[filename] + + // Create a `Render` from the currently referenced content; write it into + // the target slice. + *target = append(*target, + Render{ + Name: fmt.Sprintf("%v/%v", packKey, filename), + Content: string(content), + }, + ) + } + } } // Run satisfies the Run function of the cli.Command interface. @@ -228,13 +305,8 @@ func (c *RenderCommand) Run(args []string) int { // Iterate the rendered files and add these to the list of renders to // output. This allows errors to surface and end things without emitting // partial output and then erroring out. - - for name, renderedFile := range renderOutput.DependentRenders() { - renders = append(renders, Render{Name: formatRenderName(name), Content: renderedFile}) - } - for name, renderedFile := range renderOutput.ParentRenders() { - renders = append(renders, Render{Name: formatRenderName(name), Content: renderedFile}) - } + rangeRenders(renderOutput.DependentRenders(), &renders) + rangeRenders(renderOutput.ParentRenders(), &renders) // If the user wants to render and display the outputs template file then // render this. In the event the render returns an error, print this but do diff --git a/internal/cli/status.go b/internal/cli/status.go index 770ba7e6..8b8371e8 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -176,7 +176,7 @@ func validateStatusArgs(b *baseCommand, args []string) error { // Flags are already parsed when this function is run // Verify pack name is provided if --name flag is used if b.deploymentName != "" && len(args) == 0 { - return fmt.Errorf("--name can only be used if pack name is provided") + return errors.New("--name can only be used if pack name is provided") } return nil } diff --git a/internal/creator/templates/changelog.md b/internal/creator/templates/changelog.md index 44277dc5..2db922c4 100644 --- a/internal/creator/templates/changelog.md +++ b/internal/creator/templates/changelog.md @@ -1,4 +1,3 @@ ## Version v0.0.1 (Unreleased) Initial Release - diff --git a/internal/creator/templates/pack_helpers.tpl b/internal/creator/templates/pack_helpers.tpl index 8f758b99..8917a39b 100644 --- a/internal/creator/templates/pack_helpers.tpl +++ b/internal/creator/templates/pack_helpers.tpl @@ -1,20 +1,52 @@ -// allow nomad-pack to set the job name -[[ define "job_name" ]] -[[- if eq .my.job_name "" -]] -[[- .nomad_pack.pack.name | quote -]] -[[- else -]] -[[- .my.job_name | quote -]] -[[- end ]] -[[- end ]] +[[- /* + +# Template Helpers + +This file contains Nomad pack template helpers. Any information outside of a +`define` template action is informational and is not rendered, allowing you +to write comments and implementation details about your helper functions here. +Some helper functions are included to get you started. + +*/ -]] + +[[- /* + +## `job_name` helper + +This helper demonstrates how to use a variable value or fall back to the pack's +metadata when that value is set to a default of "". + +*/ -]] + +[[- define "job_name" -]] +[[ coalesce ( var "job_name" .) (meta "pack.name" .) | quote ]] +[[- end -]] + +[[- /* + +## `region` helper + +This helper demonstrates conditional element rendering. If your pack specifies +a variable named "region" and it's set, the region line will render otherwise +it won't. + +*/ -]] -// only deploys to a region if specified [[ define "region" -]] -[[- if not (eq .my.region "") -]] - region = [[ .my.region | quote]] +[[- if var "region" . -]] + region = "[[ var "region" . ]]" [[- end -]] [[- end -]] -// Generic constraint +[[- /* + +## `constraints` helper + +This helper creates Nomad constraint blocks from a value of type + `list(object(attribute string, operator string, value string))` + +*/ -]] + [[ define "constraints" -]] [[ range $idx, $constraint := . ]] constraint { @@ -27,25 +59,43 @@ [[ end -]] [[- end -]] -// Generic "service" block template +[[- /* + +## `service` helper + +This helper creates Nomad constraint blocks from a value of type + +``` + list( + object( + service_name string, service_port_label string, service_tags list(string), + upstreams list(object(name string, port number)) + check_type string, check_path string, check_interval string, check_timeout string + ) + ) +``` + +The template context should be set to the value of the object when calling the +template. + +*/ -]] + [[ define "service" -]] [[ $service := . ]] service { name = [[ $service.service_name | quote ]] port = [[ $service.service_port_label | quote ]] tags = [[ $service.service_tags | toStringList ]] - [[- if gt (len $service.upstreams) 0 ]] + [[- if $service.upstreams ]] connect { sidecar_service { proxy { - [[- if gt (len $service.upstreams) 0 ]] [[- range $upstream := $service.upstreams ]] upstreams { destination_name = [[ $upstream.name | quote ]] local_bind_port = [[ $upstream.port ]] } [[- end ]] - [[- end ]] } } } @@ -54,33 +104,61 @@ type = [[ $service.check_type | quote ]] [[- if $service.check_path]] path = [[ $service.check_path | quote ]] - [[- end]] + [[- end ]] interval = [[ $service.check_interval | quote ]] timeout = [[ $service.check_timeout | quote ]] } } [[- end ]] -// Generic env_vars template +[[- /* + +## `env_vars` helper + +This helper formats maps as key and quoted value pairs. + +*/ -]] + [[ define "env_vars" -]] [[- range $idx, $var := . ]] [[ $var.key ]] = [[ $var.value | quote ]] [[- end ]] [[- end ]] +[[- /* + +## `mounts` helper + +``` + list( + object( + type string, target string, source string, + readonly bool, + bind_options map(string) + ) + ) +``` + +This helper is extremely similar to the `services` helper. It uses several +alternative syntax choices and leverages the fact that range provides the +current iteratee as the current template context inside of it's scope. + +Additional notes: + "", 0, false, nil, and empty slices all evaluate to false for `if` + "", 0, false, nil, and empty slices all evaluate to empty for `range` -// Generic mount template +*/ -]] [[ define "mounts" -]] -[[- range $idx, $mount := . ]] +[[- range . ]] mount { - type = [[ $mount.type | quote ]] - target = [[ $mount.target | quote ]] - source = [[ $mount.source | quote ]] - readonly = [[ $mount.readonly ]] - [[- if gt (len $mount.bind_options) 0 ]] + type = [[ quote .type ]] + target = [[ quote .target ]] + source = [[ quote .source ]] + readonly = [[ .readonly ]] + [[- if .bind_options ]] bind_options { - [[- range $idx, $opt := $mount.bind_options ]] - [[ $opt.name ]] = [[ $opt.value | quote ]] + [[- range .bind_options ]] + [[ .name ]] = [[ quote .value ]] [[- end ]] } [[- end ]] @@ -88,7 +166,15 @@ [[- end ]] [[- end ]] -// Generic resources template +[[- /* + +## `resources` helper + +This helper formats values of object(cpu number, memory number) as a `resources` +block + +*/ -]] + [[ define "resources" -]] [[- $resources := . ]] resources { diff --git a/internal/creator/templates/pack_jobspec.tpl b/internal/creator/templates/pack_jobspec.tpl index 9d7e8348..d495a9b0 100644 --- a/internal/creator/templates/pack_jobspec.tpl +++ b/internal/creator/templates/pack_jobspec.tpl @@ -1,10 +1,10 @@ job [[ template "job_name" . ]] { [[ template "region" . ]] - datacenters = [[ .my.datacenters | toStringList ]] + datacenters = [[ var "datacenters" . | toStringList ]] type = "service" group "app" { - count = [[ .my.count ]] + count = [[ var "count" . ]] network { port "http" { @@ -12,10 +12,10 @@ job [[ template "job_name" . ]] { } } - [[ if .my.register_consul_service ]] + [[ if .register_consul_service ]] service { - name = "[[ .my.consul_service_name ]]" - tags = [[ .my.consul_service_tags | toStringList ]] + name = "[[ var "consul_service_name" . ]]" + tags = [[ var "consul_service_tags" . | toStringList ]] port = "http" check { name = "alive" @@ -43,7 +43,7 @@ job [[ template "job_name" . ]] { } env { - MESSAGE = [[.my.message | quote]] + MESSAGE = [[ var "message" . | quote ]] } } } diff --git a/internal/creator/templates/pack_metadata.hcl b/internal/creator/templates/pack_metadata.hcl index ea99977c..b255d20f 100644 --- a/internal/creator/templates/pack_metadata.hcl +++ b/internal/creator/templates/pack_metadata.hcl @@ -1,19 +1,15 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - app { url = "" } pack { name = "{{.PackName}}" description = "" - url = "" version = "" } // Optional dependency information. This block can be repeated. -// dependency { -// name = "demo_dependency_pack_name" +// dependency "demo_dep" { +// alias = "demo_dep" // source = "git://source.git/packs/demo_dependency_pack" // } diff --git a/internal/creator/templates/pack_variables.hcl b/internal/creator/templates/pack_variables.hcl index 72311f4b..2d2331ad 100644 --- a/internal/creator/templates/pack_variables.hcl +++ b/internal/creator/templates/pack_variables.hcl @@ -1,11 +1,8 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: MPL-2.0 - variable "job_name" { + # If "", the pack name will be used description = "The name to use as the job name which overrides using the pack name" type = string - // If "", the pack name will be used - default = "" + default = "" } variable "region" { @@ -33,25 +30,25 @@ variable "message" { } variable "register_consul_service" { - description = "If you want to register a consul service for the job" + description = "If you want to register a Consul service for the job" type = bool default = true } variable "consul_service_name" { - description = "The consul service name for the {{.PackName}} application" + description = "The Consul service name for the {{.PackName}} application" type = string default = "webapp" } variable "consul_service_tags" { - description = "The consul service name for the {{.PackName}} application" + description = "The Consul service name for the {{.PackName}} application" type = list(string) - // defaults to integrate with Fabio or Traefik - // This routes at the root path "/", to route to this service from - // another path, change "urlprefix-/" to "urlprefix-/" and - // "traefik.http.routers.http.rule=Path(∫/∫)" to - // "traefik.http.routers.http.rule=Path(∫/∫)" + # The default value is shaped to integrate with Fabio or Traefik + # This routes at the root path "/", to route to this service from + # another path, change "urlprefix-/" to "urlprefix-/" and + # "traefik.http.routers.http.rule=Path(∫/∫)" to + # "traefik.http.routers.http.rule=Path(∫/∫)" default = [ "urlprefix-/", "traefik.enable=true", diff --git a/internal/pkg/cache/cache_test.go b/internal/pkg/cache/cache_test.go index 8cd2ed11..fa86c5f2 100644 --- a/internal/pkg/cache/cache_test.go +++ b/internal/pkg/cache/cache_test.go @@ -21,6 +21,8 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/errors" "github.com/hashicorp/nomad-pack/internal/pkg/helper/filesystem" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" + "github.com/hashicorp/nomad/ci" ) var ( @@ -364,7 +366,7 @@ func TestDeletePackByRef(t *testing.T) { } func TestParsePackURL(t *testing.T) { - t.Parallel() + ci.Parallel(t) reg := &Registry{} testCases := []struct { @@ -401,6 +403,7 @@ func TestParsePackURL(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) ok := reg.parsePackURL(tc.path) t.Logf(" path: %s\nsource: %s\n ok: %v\n\n", tc.path, reg.Source, ok) if tc.expectOk { @@ -472,12 +475,12 @@ func NewTestLogger(t *testing.T) *TestLogger { // a parameter. type NoopLogger struct{} -func (_ NoopLogger) Debug(_ string) {} -func (_ NoopLogger) Error(_ string) {} -func (_ NoopLogger) ErrorWithContext(_ error, _ string, _ ...string) {} -func (_ NoopLogger) Info(_ string) {} -func (_ NoopLogger) Trace(_ string) {} -func (_ NoopLogger) Warning(_ string) {} +func (NoopLogger) Trace(string) {} +func (NoopLogger) Debug(string) {} +func (NoopLogger) Info(string) {} +func (NoopLogger) Warning(string) {} +func (NoopLogger) Error(string) {} +func (NoopLogger) ErrorWithContext(error, string, ...string) {} type TestGithubRegistry struct { sourceURL string @@ -530,7 +533,7 @@ func makeTestRegRepo(tReg *TestGithubRegistry) { tReg.cleanupFn = func() { os.RemoveAll(tReg.tmpDir) } tReg.sourceURL = path.Join(tReg.tmpDir, "test_registry.git") - err = filesystem.CopyDir("../../../fixtures/test_registry", tReg.SourceURL(), false, NoopLogger{}) + err = filesystem.CopyDir(testfixture.MustAbsPath("v2/test_registry"), tReg.SourceURL(), false, NoopLogger{}) if err != nil { tReg.Cleanup() panic(fmt.Errorf("unable to copy test fixtures to test git repo: %v", err)) @@ -704,7 +707,7 @@ func listAllTestPacks(t *testing.T, cachePath string) packtuples { t.Fatalf("listAllTestPacks: WalkDir error: %v", err) } - if d.IsDir() { + if testing.Verbose() && d.IsDir() { t.Logf("walking %q...", p) } diff --git a/internal/pkg/deps/vendor.go b/internal/pkg/deps/vendor.go index fcfe0753..d12e55bd 100644 --- a/internal/pkg/deps/vendor.go +++ b/internal/pkg/deps/vendor.go @@ -5,6 +5,7 @@ package deps import ( "context" + "errors" "fmt" "path" @@ -26,7 +27,7 @@ func Vendor(ctx context.Context, ui terminal.UI, targetPath string) error { } if len(metadata.Dependencies) == 0 { - return fmt.Errorf("metadata.hcl file does not contain any dependencies") + return errors.New("metadata.hcl file does not contain any dependencies") } for _, d := range metadata.Dependencies { diff --git a/internal/pkg/deps/vendor_test.go b/internal/pkg/deps/vendor_test.go index b741c46b..3de6daba 100644 --- a/internal/pkg/deps/vendor_test.go +++ b/internal/pkg/deps/vendor_test.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/helper" "github.com/hashicorp/nomad-pack/internal/pkg/helper/filesystem" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" "github.com/hashicorp/nomad-pack/internal/testui" "github.com/hashicorp/nomad-pack/sdk/pack" ) @@ -90,7 +91,7 @@ func TestVendor(t *testing.T) { } tmpDependencySourceDir2 := t.TempDir() - must.NoError(t, createTestDepRepo(tmpDependencySourceDir2)) + must.NoError(t, createTestDepRepo(t, tmpDependencySourceDir2)) goodMetadata.Dependencies[0].Source = path.Join( tmpDependencySourceDir2, "simple_raw_exec", @@ -110,9 +111,10 @@ func TestVendor(t *testing.T) { } // createTestDepRepo creates a git repository with a dependency pack in it -func createTestDepRepo(dst string) error { +func createTestDepRepo(t *testing.T, dst string) error { + err := filesystem.CopyDir( - "../../../fixtures/test_registry/packs/simple_raw_exec", + testfixture.AbsPath(t, "v2/test_registry/packs/simple_raw_exec"), path.Join(dst, "simple_raw_exec"), false, NoopLogger{}, diff --git a/internal/pkg/errors/packdiags/packdiags.go b/internal/pkg/errors/packdiags/packdiags.go new file mode 100644 index 00000000..c7d4ea2d --- /dev/null +++ b/internal/pkg/errors/packdiags/packdiags.go @@ -0,0 +1,91 @@ +package packdiags + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/helper" +) + +// DiagFileNotFound is returned when pack parsing encounters a required file +// that is missing. +func DiagFileNotFound(f string) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to read file", + Detail: fmt.Sprintf("The file %q could not be read.", f), + } +} + +// DiagMissingRootVar is returned when a pack consumer passes in a variable that +// is not defined for the pack. +func DiagMissingRootVar(name string, sub *hcl.Range) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing base variable declaration to override", + Detail: fmt.Sprintf(`There is no variable named %q. An override file can only override a variable that was already declared in a primary configuration file.`, name), + Subject: sub, + } +} + +// DiagInvalidDefaultValue is returned when the default for a variable does not +// match the specified variable type. +func DiagInvalidDefaultValue(detail string, sub *hcl.Range) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid default value for variable", + Detail: detail, + Subject: sub, + } +} + +// DiagFailedToConvertCty is an error that can happen late in parsing. It should +// not occur, but is here for coverage. +func DiagFailedToConvertCty(err error, sub *hcl.Range) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to convert Cty to interface", + Detail: helper.Title(err.Error()), + Subject: sub, + } +} + +// DiagInvalidValueForType is returned when a pack consumer attempts to set a +// variable to an inappopriate value based on the pack's variable specification +func DiagInvalidValueForType(err error, sub *hcl.Range) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid value for variable", + Detail: fmt.Sprintf("This variable value is not compatible with the variable's type constraint: %s.", err), + Subject: sub, + } +} + +// DiagInvalidVariableName is returned when a pack author specifies an invalid +// name for a variable in their varfile +func DiagInvalidVariableName(sub *hcl.Range) *hcl.Diagnostic { + return &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable name", + Detail: "Name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.", + Subject: sub, + } +} + +// SafeDiagnosticsAppend prevents a nil Diagnostic from appending to the target +// Diagnostics, since HasError is not nil-safe. +func SafeDiagnosticsAppend(base hcl.Diagnostics, in *hcl.Diagnostic) hcl.Diagnostics { + if in != nil { + base = base.Append(in) + } + return base +} + +// SafeDiagnosticsExtend clean where the input Diagnostics of nils as they are +// appended to the base +func SafeDiagnosticsExtend(base, in hcl.Diagnostics) hcl.Diagnostics { + for _, diag := range in { + base = SafeDiagnosticsAppend(base, diag) + } + return base +} diff --git a/internal/pkg/errors/packdiags/packdiags_test.go b/internal/pkg/errors/packdiags/packdiags_test.go new file mode 100644 index 00000000..76c79f7e --- /dev/null +++ b/internal/pkg/errors/packdiags/packdiags_test.go @@ -0,0 +1,130 @@ +package packdiags + +import ( + "errors" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad/ci" + "github.com/shoenig/test/must" + "github.com/stretchr/testify/require" +) + +var testRange = hcl.Range{ + Filename: "«filename»", + Start: hcl.Pos{ + Line: 1, + Column: 1, + Byte: 0, + }, + End: hcl.Pos{ + Line: 1, + Column: 6, + Byte: 5, + }, +} + +func TestPackDiag_DiagFileNotFound(t *testing.T) { + ci.Parallel(t) + diag := DiagFileNotFound("test.txt") + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Failed to read file", diag.Summary) + must.Eq(t, `The file "test.txt" could not be read.`, diag.Detail) + + diags := make(hcl.Diagnostics, 0, 1) + diags = SafeDiagnosticsAppend(diags, diag) + must.True(t, diags.HasErrors()) +} + +func TestPackDiag_DiagMissingRootVar(t *testing.T) { + ci.Parallel(t) + diag := DiagMissingRootVar("myVar", &testRange) + + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Missing base variable declaration to override", diag.Summary) + must.Eq(t, `There is no variable named "myVar". An override file can only override a variable that was already declared in a primary configuration file.`, diag.Detail) + must.Eq(t, testRange, *diag.Subject) +} + +func TestPackDiag_DiagInvalidDefaultValue(t *testing.T) { + ci.Parallel(t) + diag := DiagInvalidDefaultValue("test detail", &testRange) + + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Invalid default value for variable", diag.Summary) + must.Eq(t, `test detail`, diag.Detail) + must.Eq(t, testRange, *diag.Subject) +} + +func TestPackDiag_DiagFailedToConvertCty(t *testing.T) { + ci.Parallel(t) + diag := DiagFailedToConvertCty(errors.New("test error"), &testRange) + + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Failed to convert Cty to interface", diag.Summary) + must.Eq(t, `Test Error`, diag.Detail) + must.Eq(t, testRange, *diag.Subject) +} +func TestPackDiag_DiagInvalidValueForType(t *testing.T) { + ci.Parallel(t) + diag := DiagInvalidValueForType(errors.New("test error"), &testRange) + + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Invalid value for variable", diag.Summary) + must.StrContains(t, diag.Detail, "This variable value is not compatible with the variable's type constraint") + must.StrContains(t, diag.Detail, "test error") + must.Eq(t, testRange, *diag.Subject) +} + +func TestPackDiag_DiagInvalidVariableName(t *testing.T) { + ci.Parallel(t) + diag := DiagInvalidVariableName(&testRange) + + must.Eq(t, diag.Severity, hcl.DiagError) + must.Eq(t, "Invalid variable name", diag.Summary) + must.Eq(t, "Name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.", diag.Detail) + must.Eq(t, testRange, *diag.Subject) +} + +func TestPackDiag_SafeDiagnosticsAppend(t *testing.T) { + diags := hcl.Diagnostics{} + var diag *hcl.Diagnostic + + // HasErrors on hcl.Diagnostics is not nil-safe + require.Panicsf(t, func() { + d1 := diags.Append(diag) + d1.HasErrors() + }, "should have panicked") + + // Using SafeDiagnosticsAppend should prevent the panic + require.NotPanicsf(t, func() { + d2 := SafeDiagnosticsAppend(diags, diag) + d2.HasErrors() + }, "should not have panicked") + + // Verify that the original case still panics + require.Panicsf(t, func() { + d1 := diags.Append(diag) + d1.HasErrors() + }, "should have panicked") +} + +func TestPackDiag_SafeDiagnosticsExtend(t *testing.T) { + var diag *hcl.Diagnostic + diags := hcl.Diagnostics{} + diags2 := hcl.Diagnostics{diag} + require.Panicsf(t, func() { + d := diags.Extend(diags2) + d.HasErrors() + }, "should have panicked") + + require.NotPanicsf(t, func() { + d := SafeDiagnosticsExtend(diags, diags2) + d.HasErrors() + }, "should not have panicked") + + require.Panicsf(t, func() { + d := diags.Extend(diags2) + d.HasErrors() + }, "should have still panicked") +} diff --git a/internal/pkg/errors/template_errors.go b/internal/pkg/errors/template_errors.go new file mode 100644 index 00000000..1aa428cc --- /dev/null +++ b/internal/pkg/errors/template_errors.go @@ -0,0 +1,229 @@ +package errors + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "text/template" + + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" +) + +// PackTemplateErrors are designed to progressively enhance the errors returned +// from go template rendering. Implements `error` +type PackTemplateError struct { + Filename string // template filename returned + Line int // the line in the template if found + StartChar int // the character number in the template if found + EndChar int // the character number calculated as the end if an "at" is found + Err error // the last element in given error text when split by ": " + Details string // some additional help text for specific known error patterns + Suggestions []string // some suggestions to add to the error context + Extra []string // remaining splits between the beginning elements and the last one which is the error + + origErr error + badElement string + at string + tplctx parser.PackTemplateContext +} + +// Error implements the `error` interface using a value receiver so it works +// with PackTemplateError values or pointers. +func (p PackTemplateError) Error() string { + return p.Err.Error() +} + +// ParseTemplateError returns a PackTemplate error that wraps and attempts to +// enhance errors returned from go template. +func ParseTemplateError(tplCtx parser.PackTemplateContext, err error) *PackTemplateError { + out := &PackTemplateError{Err: err, origErr: err, tplctx: tplCtx} + + var execErr template.ExecError + if errors.As(err, &execErr) { + out.parseExecError(execErr) + } + + return out +} + +// ToWrappedUIContext converts a PackTemplateError into a WrappedUIContext for +// display to the CLI +func (p *PackTemplateError) ToWrappedUIContext() *WrappedUIContext { + errCtx := NewUIErrorContext() + errCtx.Add("Details: ", p.Details) + errCtx.Add("Filename: ", p.Filename) + errCtx.Add("Position: ", p.pos()) + errCtx.Add("Suggestions: ", strings.Join(p.Suggestions, "; ")) + return &WrappedUIContext{ + Err: p, + Subject: "error executing template", + Context: errCtx, + } +} + +func (p *PackTemplateError) pos() string { + var out string + if p.Line == 0 { + return "" + } + out = fmt.Sprint(p.Line) + if p.StartChar != 0 { + out += fmt.Sprintf(",%d", p.StartChar) + } + return out +} + +// parseExecError attempts to decode the textual representation of a go +// template ExecError. To quote the Go text/template source: +// +// > "TODO: It would be nice if ExecError was more broken down, but +// the way ErrorContext embeds the template name makes the +// processing too clumsy." +func (p *PackTemplateError) parseExecError(execErr template.ExecError) { + // Exchange wrapped error for unwrapped one, in case we bail out early + p.Err = execErr + p.Filename = execErr.Name + + // If there is a source at the beginning of the error, enhanceSource will + // parse it into the struct and then pop it off. + p.extractSource() + + // We should be able to split off the last `: ` and have it be the error proper. + if parts := strings.Split(p.Err.Error(), ": "); len(parts) > 1 { + p.Extra = parts + } + + // Tee up a reasonable error value. We'll try to enhance it, but on any error + // after here we'll return this since it's "good enough" + p.Err = errors.New(p.Extra[len(p.Extra)-1]) + + // Maybe we can do better on the "variable.PackContextable" bit if it shows up + if strings.Contains(p.Err.Error(), "variable.PackContextable") { + p.fixupPackContextable() + } + + p.enhance() +} + +func (p *PackTemplateError) extractSource() { + hasElement := true + in := p.Err.Error() + + for hasElement { + if b, a, found := strings.Cut(in, ": "); found { + // the first element is "template" + if b == "template" { + in = a + continue + } + // the filename component might have line and character details + if parts := strings.Split(b, ":"); len(parts) > 1 { + if len(parts) == 3 { + if charInt, err := strconv.Atoi(parts[2]); err == nil { + p.StartChar = charInt + } + } + if len(parts) >= 2 { + if lineInt, err := strconv.Atoi(parts[1]); err == nil { + p.Line = lineInt + } + } + p.Filename = parts[0] + in = a + continue + } + hasElement = false + } + } +} + +func (p *PackTemplateError) enhance() { + if p.isNPE() { + p.enhanceNPE() + } +} + +func (p *PackTemplateError) isNPE() bool { + return strings.HasPrefix(p.Err.Error(), "nil pointer") +} + +func (p *PackTemplateError) enhanceNPE() { + // Nil pointer exceptions could have some _somewhat_ common reasons + if strings.HasPrefix(p.badElement, ".") { + // If they are trying to access a context element starting with a dot + // directly, then they are probably using the old syntax. Since I'm not + // 100% on this yet, I'm going to reorganize the error message a little + // and add some information to DidYouMean + p.Err = fmt.Errorf("Pack %q not found when accessing %q", strings.TrimPrefix(p.badElement, "."), p.at) + p.Details = "The referenced pack was not found in the template context." + if parts := strings.Split(p.at, "."); len(parts) == 3 && parts[0] == "" { + // This case very much looks like the old .packname.varname. Let's try to + // make a sugestion with the details. + atPackName := parts[1] + rootPackName := p.tplctx.Name() + + if atPackName == rootPackName { + atPackName = "" + } + + p.Suggestions = []string{ + fmt.Sprintf( + "The legacy %q syntax should be updated to use `var %q .%s`.", + p.at, + parts[2], + atPackName, + ), + } + } + + } +} + +func (p *PackTemplateError) fixupPackContextable() { + const typeConst = "variable.PackContextable" + errStr := p.Err.Error() + + // attempt to extract the "variable.PackContextable.blah" part. + varRefStr := "" + + // Since there's no variable component to the regex, we can use MustCompile + pRE := regexp.MustCompile(`(?m)^.*(variable\.PackContextable\.[\w]+)(?:[[:space:]]|$)`) + + // If there's a match, FindStringSubmatch will return 2 matches: one for the + // whole string and one for the capture group + if matches := pRE.FindStringSubmatch(errStr); len(matches) == 2 { + varRefStr = strings.TrimPrefix(matches[1], typeConst) + } else { + // If we can't extract the variable reference from the error, then bail out + return + } + + atRE, err := regexp.Compile(`^.*"` + p.Filename + `" at <(.+)>$`) + if err != nil { + // there's a janky filename that's breaking the regex. + return + } + + for _, e := range p.Extra { + // if it matches, there should be 2 items in the matches slice + if matches := atRE.FindStringSubmatch(e); len(matches) == 2 { + // We have an "at" token value at this point. This should provide some + // context with which to replace the text "variable.PackContextable", + // potentially other type values too. + p.at = matches[1] + p.EndChar = p.StartChar + len(p.at) + + // At this point, we should have the variable reference and an at + // reference. The combination of these two items _should_ be able + // to make a rational description of what they were accessing + // without exposing the variable.PackContextable type component. + if bad, _, found := strings.Cut(p.at, varRefStr); found { + p.badElement = bad + break + } + } + } + p.Err = errors.New(strings.ReplaceAll(errStr, typeConst+varRefStr, p.badElement)) +} diff --git a/internal/pkg/helper/filesystem/filesystem.go b/internal/pkg/helper/filesystem/filesystem.go index 6fcc2c94..a545c0f7 100644 --- a/internal/pkg/helper/filesystem/filesystem.go +++ b/internal/pkg/helper/filesystem/filesystem.go @@ -91,7 +91,7 @@ func CopyDir(sourceDir string, destinationDir string, overwrite bool, logger log // Throw error if not a directory // TODO: Might need to handle symlinks. if !sourceDirInfo.IsDir() { - err = fmt.Errorf("source is not a directory") + err = errors.New("source is not a directory") logger.Debug(err.Error()) return } @@ -106,7 +106,7 @@ func CopyDir(sourceDir string, destinationDir string, overwrite bool, logger log if !overwrite { // throw error if it does exist if err == nil { - err = fmt.Errorf("destination already exists") + err = errors.New("destination already exists") logger.Debug(err.Error()) return } diff --git a/internal/pkg/manager/manager.go b/internal/pkg/manager/manager.go index 928487de..8504fbb9 100644 --- a/internal/pkg/manager/manager.go +++ b/internal/pkg/manager/manager.go @@ -11,7 +11,8 @@ import ( "github.com/hashicorp/nomad-pack/internal/pkg/errors" "github.com/hashicorp/nomad-pack/internal/pkg/loader" "github.com/hashicorp/nomad-pack/internal/pkg/renderer" - "github.com/hashicorp/nomad-pack/internal/pkg/variable" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" "github.com/hashicorp/nomad-pack/sdk/pack" "github.com/hashicorp/nomad/api" ) @@ -23,6 +24,7 @@ type Config struct { VariableFiles []string VariableCLIArgs map[string]string VariableEnvVars map[string]string + UseParserV1 bool } // PackManager is responsible for loading, parsing, and rendering a Pack and @@ -43,8 +45,10 @@ func NewPackManager(cfg *Config, client *api.Client) *PackManager { } } -// TODO: This shares a ton of code with ProcessTemplates and is a good refactoring target -func (pm *PackManager) ProcessVariableFiles() (*variable.ParsedVariables, []*errors.WrappedUIContext) { +// ProcessVariableFiles creates the map of packs to their respective variables +// definition files. This is used between the variable override file generator +// code and the ProcessTemplates logic in this file. +func (pm *PackManager) ProcessVariableFiles() (*parser.ParsedVariables, []*errors.WrappedUIContext) { loadedPack, err := pm.loadAndValidatePacks() if err != nil { return nil, []*errors.WrappedUIContext{{ @@ -56,19 +60,25 @@ func (pm *PackManager) ProcessVariableFiles() (*variable.ParsedVariables, []*err pm.loadedPack = loadedPack - // Root vars are nested under the parent pack name, which is currently - // just the pack name without the version. We want to slice the string - // so it's just the pack name without the version - parentName := path.Base(pm.cfg.Path) - idx := strings.LastIndex(path.Base(pm.cfg.Path), "@") - if idx != -1 { - parentName = path.Base(pm.cfg.Path)[0:idx] - } - variableParser, err := variable.NewParser(&variable.ParserConfig{ - ParentName: parentName, + // Root vars are nested under the pack name, which is currently the pack name + // without the version. + parentName, _, _ := strings.Cut(path.Base(pm.cfg.Path), "@") + + pCfg := &config.ParserConfig{ + Version: config.V2, + ParentPackID: pack.ID(parentName), RootVariableFiles: loadedPack.RootVariableFiles(), - }) + EnvOverrides: pm.cfg.VariableEnvVars, + FileOverrides: pm.cfg.VariableFiles, + FlagOverrides: pm.cfg.VariableCLIArgs, + } + + if pm.cfg.UseParserV1 { + pCfg.Version = config.V1 + pCfg.ParentName = parentName + } + variableParser, err := parser.NewParser(pCfg) if err != nil { return nil, []*errors.WrappedUIContext{{ Err: err, @@ -85,57 +95,23 @@ func (pm *PackManager) ProcessVariableFiles() (*variable.ParsedVariables, []*err return parsedVars, nil } -// ProcessTemplates is responsible for running all backend process for the PackManager -// returning an error along with the ProcessedPack. This contains all the -// rendered templates. +// ProcessTemplates is responsible for running all backend process for the +// PackManager returning an error along with the ProcessedPack. This contains +// all the rendered templates. // // TODO(jrasell) figure out whether we want an error or hcl.Diagnostics return // object. If we stick to an error, then we need to come up with a way of // nicely formatting them. func (pm *PackManager) ProcessTemplates(renderAux bool, format bool, ignoreMissingVars bool) (*renderer.Rendered, []*errors.WrappedUIContext) { - loadedPack, err := pm.loadAndValidatePacks() - if err != nil { - return nil, []*errors.WrappedUIContext{{ - Err: err, - Subject: "failed to validate packs", - Context: errors.NewUIErrorContext(), - }} - } - - pm.loadedPack = loadedPack - - // Root vars are nested under the parent pack name, which is currently - // just the pack name without the version. We want to slice the string - // so it's just the pack name without the version - parentName := path.Base(pm.cfg.Path) - idx := strings.LastIndex(path.Base(pm.cfg.Path), "@") - if idx != -1 { - parentName = path.Base(pm.cfg.Path)[0:idx] - } - - variableParser, err := variable.NewParser(&variable.ParserConfig{ - ParentName: parentName, - RootVariableFiles: loadedPack.RootVariableFiles(), - FileOverrides: pm.cfg.VariableFiles, - CLIOverrides: pm.cfg.VariableCLIArgs, - EnvOverrides: pm.cfg.VariableEnvVars, - IgnoreMissingVars: ignoreMissingVars, - }) - if err != nil { - return nil, []*errors.WrappedUIContext{{ - Err: err, - Subject: "failed to instantiate parser", - Context: errors.NewUIErrorContext(), - }} - } - - parsedVars, diags := variableParser.Parse() - if diags != nil && diags.HasErrors() { - return nil, errors.HCLDiagsToWrappedUIContext(diags) + parsedVars, wErr := pm.ProcessVariableFiles() + if wErr != nil { + return nil, wErr } - mapVars, diags := parsedVars.ConvertVariablesToMapInterface() + // Pre-test the parsed variables so that we can trust them + // in rendering and to use for errors later + tplCtx, diags := parsedVars.ToPackTemplateContext(pm.loadedPack) if diags != nil && diags.HasErrors() { return nil, errors.HCLDiagsToWrappedUIContext(diags) } @@ -150,13 +126,11 @@ func (pm *PackManager) ProcessTemplates(renderAux bool, format bool, ignoreMissi // should we format before rendering? pm.renderer.Format = format - rendered, err := r.Render(loadedPack, mapVars) + rendered, err := r.Render(pm.loadedPack, parsedVars) if err != nil { - return nil, []*errors.WrappedUIContext{{ - Err: err, - Subject: "failed to instantiate parser", - Context: errors.NewUIErrorContext(), - }} + return nil, []*errors.WrappedUIContext{ + errors.ParseTemplateError(tplCtx, err).ToWrappedUIContext(), + } } return rendered, nil } @@ -190,32 +164,33 @@ func (pm *PackManager) loadAndValidatePacks() (*pack.Pack, error) { return parentPack, nil } -// loadAndValidatePack recursively loads a pack and it's dependencies. Errors +// loadAndValidatePack recursively loads a pack and its dependencies. Errors // result in an immediate return. func (pm *PackManager) loadAndValidatePack(cur *pack.Pack, depsPath string) error { - for _, dependency := range cur.Metadata.Dependencies { + for _, dep := range cur.Metadata.Dependencies { // Skip any dependencies that are not enabled. - if !*dependency.Enabled { + if !*dep.Enabled { continue } - // Load and validate the dependent pack. - dependentPack, err := loader.Load(path.Join(depsPath, dependency.Name)) + // Load and validate the dependency pack. + packPath := path.Join(depsPath, path.Clean(dep.Name)) + depPack, err := loader.Load(packPath) if err != nil { return fmt.Errorf("failed to load dependent pack: %v", err) } - if err := dependentPack.Validate(); err != nil { + if err := depPack.Validate(); err != nil { return fmt.Errorf("failed to validate dependent pack: %v", err) } // Add the dependency to the current pack. - cur.AddDependencies(dependentPack) + cur.AddDependency(dep.ID(), depPack) // Recursive call. - if err := pm.loadAndValidatePack(dependentPack, depsPath); err != nil { + if err := pm.loadAndValidatePack(depPack, path.Join(packPath, "deps")); err != nil { return err } } diff --git a/internal/pkg/renderer/funcs.go b/internal/pkg/renderer/funcs.go index fb987d2e..127802ed 100644 --- a/internal/pkg/renderer/funcs.go +++ b/internal/pkg/renderer/funcs.go @@ -6,19 +6,28 @@ package renderer import ( "fmt" "os" + "strings" "text/template" "github.com/Masterminds/sprig/v3" "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" "github.com/hashicorp/nomad/api" + "golang.org/x/exp/maps" ) // funcMap instantiates our default template function map with populated // functions for use within text.Template. -func funcMap(nomadClient *api.Client) template.FuncMap { +func funcMap(r *Renderer) template.FuncMap { - // Sprig defines our base map. - f := sprig.TxtFuncMap() + // The base of the funcmap comes from the template context funcs + f := make(template.FuncMap) + if r != nil && r.pv != nil { + maps.Copy(f, parser.PackTemplateContextFuncs(r.pv.IsV1())) + } + + // Copy the sprig funcs into the funcmap. + maps.Copy(f, sprig.TxtFuncMap()) // Add debugging functions. These are useful when debugging templates and // variables. @@ -36,10 +45,10 @@ func funcMap(nomadClient *api.Client) template.FuncMap { f["withSortKeys"] = withSortKeys f["withSpewKeys"] = withSpewKeys - if nomadClient != nil { - f["nomadNamespaces"] = nomadNamespaces(nomadClient) - f["nomadNamespace"] = nomadNamespace(nomadClient) - f["nomadRegions"] = nomadRegions(nomadClient) + if r != nil && r.Client != nil { + f["nomadNamespaces"] = nomadNamespaces(r.Client) + f["nomadNamespace"] = nomadNamespace(r.Client) + f["nomadRegions"] = nomadRegions(r.Client) } // Add additional custom functions. @@ -84,15 +93,25 @@ func nomadRegions(client *api.Client) func() ([]string, error) { // toStringList takes a list of string and returns the HCL equivalent which is // useful when templating jobs and params such as datacenters. -func toStringList(l []any) (string, error) { - var out string - for i := range l { - if i > 0 && i < len(l) { - out += ", " +func toStringList(l any) (string, error) { + var out strings.Builder + out.WriteRune('[') + switch tl := l.(type) { + case []any: + // If l is a []string, then the caller probably wants that printed + // as a list of quoted elements, JSON style. + for i, v := range tl { + if i > 0 { + out.WriteString(", ") + } + out.WriteString(fmt.Sprintf("%q", v)) } - out += fmt.Sprintf("%q", l[i]) + default: + out.WriteString(fmt.Sprintf("%q", l)) } - return "[" + out + "]", nil + out.WriteRune(']') + o := out.String() + return o, nil } // Spew helper funcs diff --git a/internal/pkg/renderer/renderer.go b/internal/pkg/renderer/renderer.go index ce020b73..747c528d 100644 --- a/internal/pkg/renderer/renderer.go +++ b/internal/pkg/renderer/renderer.go @@ -9,12 +9,16 @@ import ( "strings" "text/template" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser" "github.com/hashicorp/nomad-pack/sdk/pack" ) +type PackTemplateContext = parser.PackTemplateContext + // Renderer provides template rendering functionality using the text/template // package. type Renderer struct { @@ -38,18 +42,34 @@ type Renderer struct { // stores the pack information, variables and tpl, so we can perform the // output template rendering after pack deployment. - pack *pack.Pack - variables map[string]any - tpl *template.Template + pack *pack.Pack + tpl *template.Template + pv *parser.ParsedVariables } -// toRender details an individual template to render along with it's scoped +// toRender details an individual template to render along with its scoped // variables. type toRender struct { content string + tplCtx PackTemplateContext variables map[string]any } +// getDot is an ugly convenience function to deal with +// the fact that ParserV1 and ParserV2 create differently +// shaped contexts. Template is very forgiving with the +// data (or dot) object's typing. +func (t toRender) getDot() any { + var out any + if len(t.variables) > 0 { + out = t.variables + } + if len(t.tplCtx) > 0 { + out = t.tplCtx + } + return out +} + const ( leftTemplateDelim = "[[" rightTemplateDelim = "]]" @@ -57,16 +77,22 @@ const ( // Render is responsible for iterating the pack and rendering each defined // template using the parsed variable map. -func (r *Renderer) Render(p *pack.Pack, variables map[string]any) (*Rendered, error) { +func (r *Renderer) Render(p *pack.Pack, variables *parser.ParsedVariables) (*Rendered, error) { + + // save the ParsedVariables into the renderer state + r.pv = variables // filesToRender stores all the templates and auxiliary files that should be // rendered filesToRender := map[string]toRender{} - prepareFiles(p, filesToRender, variables, r.RenderAuxFiles) + err := r.prepareFiles(p, filesToRender, variables, r.RenderAuxFiles) + if err != nil { + return nil, err + } // Set up our new template, add the function mapping, and set the // delimiters. - tpl := template.New("tpl").Funcs(funcMap(r.Client)).Delims(leftTemplateDelim, rightTemplateDelim) + tpl := template.New("tpl").Funcs(funcMap(r)).Delims(leftTemplateDelim, rightTemplateDelim) // Control the behaviour of rendering when it encounters an element // referenced which doesn't exist within the variable mapping. @@ -86,8 +112,8 @@ func (r *Renderer) Render(p *pack.Pack, variables map[string]any) (*Rendered, er // Generate our output structure. rendered := &Rendered{ - parentRenders: make(map[string]string), - dependentRenders: make(map[string]string), + parentRenders: make(map[string]string), + dependencyRenders: make(map[string]string), } for name, src := range filesToRender { @@ -102,7 +128,8 @@ func (r *Renderer) Render(p *pack.Pack, variables map[string]any) (*Rendered, er // is an error. var buf strings.Builder - if err := tpl.ExecuteTemplate(&buf, name, src.variables); err != nil { + dot := src.getDot() + if err := tpl.ExecuteTemplate(&buf, name, dot); err != nil { return nil, fmt.Errorf("failed to render %s: %v", name, err) } @@ -120,24 +147,25 @@ func (r *Renderer) Render(p *pack.Pack, variables map[string]any) (*Rendered, er continue } - if r.Format { + if r.Format && + (strings.HasSuffix(name, ".nomad.tpl") || strings.HasSuffix(name, ".hcl.tpl")) { // hclfmt the templates f := hclwrite.Format([]byte(replacedTpl)) replacedTpl = string(f) } // Add the rendered pack template to our output, depending on whether - // it's name matches that of our parent. + // its name matches that of our parent. if nameSplit[0] == p.Name() { rendered.parentRenders[name] = replacedTpl } else { - rendered.dependentRenders[name] = replacedTpl + rendered.dependencyRenders[name] = replacedTpl } } - r.variables = variables r.pack = p r.tpl = tpl + r.pv = variables return rendered, nil } @@ -155,17 +183,70 @@ func (r *Renderer) RenderOutput() (string, error) { } var buf strings.Builder - if err := r.tpl.ExecuteTemplate(&buf, r.pack.OutputTemplateFile.Name, r.variables); err != nil { + if err := r.tpl.ExecuteTemplate(&buf, r.pack.OutputTemplateFile.Name, r.pv); err != nil { return "", fmt.Errorf("failed to render %s: %v", r.pack.OutputTemplateFile.Name, err) } return buf.String(), nil } -// prepareFiles recurses the pack and its dependencies and returns a map +// prepareFiles dispatches the request to prepare the Renderer's file configs +// to the parser version specific implementation +func (r *Renderer) prepareFiles(p *pack.Pack, + files map[string]toRender, + variables *parser.ParsedVariables, + renderAuxFiles bool, +) hcl.Diagnostics { + + if variables.IsV1() { + v1TplCtx, diags := variables.ConvertVariablesToMapInterface() + if diags.HasErrors() { + return diags + } + prepareFilesV1(p, files, v1TplCtx, renderAuxFiles) + return nil + } + + v2TplCtx, diags := variables.ToPackTemplateContext(p) + if diags.HasErrors() { + return diags + } + prepareFilesV2(p, files, v2TplCtx, renderAuxFiles) + return nil +} + +// prepareFilesV2 recurses the pack and its dependencies and returns a map // with the templates/auxiliary files to render along with the variables which // correspond. -func prepareFiles(p *pack.Pack, +func prepareFilesV2(p *pack.Pack, + files map[string]toRender, + tplCtx parser.PackTemplateContext, + renderAuxFiles bool, +) { + + // Iterate the dependencies and prepareTemplates for each. + for _, child := range p.Dependencies() { + prepareFilesV2(child, files, tplCtx[child.AliasOrName()].(PackTemplateContext), renderAuxFiles) + } + + // Add each template within the pack with scoped variables. + for _, t := range p.TemplateFiles { + files[path.Join(p.VariablesPath().AsPath(), t.Name)] = toRender{content: string(t.Content), tplCtx: tplCtx} + } + + if renderAuxFiles { + // Add each aux file within the pack with scoped variables. + for _, f := range p.AuxiliaryFiles { + files[path.Join(p.VariablesPath().AsPath(), f.Name)] = toRender{content: string(f.Content), tplCtx: tplCtx} + } + } +} + +// prepareFilesV1 recurses the pack and its dependencies and returns a map +// with the templates/auxiliary files to render along with the variables which +// correspond. It is retained so that users can fall back to the original +// template contexts as they migrate their packs to the newer syntax. +func prepareFilesV1(p *pack.Pack, files map[string]toRender, variables map[string]any, renderAuxFiles bool, @@ -194,7 +275,7 @@ func prepareFiles(p *pack.Pack, } // Iterate the dependencies and prepareTemplates for each. for _, child := range p.Dependencies() { - prepareFiles(child, files, newVars, renderAuxFiles) + prepareFilesV1(child, files, newVars, renderAuxFiles) } // Add each template within the pack with scoped variables. @@ -214,8 +295,8 @@ func prepareFiles(p *pack.Pack, // pack. It splits them based on whether they belong to the parent or a // dependency. type Rendered struct { - parentRenders map[string]string - dependentRenders map[string]string + parentRenders map[string]string + dependencyRenders map[string]string } // ParentRenders returns a map of rendered templates belonging to the parent @@ -229,8 +310,8 @@ func (r *Rendered) LenParentRenders() int { return len(r.parentRenders) } // DependentRenders returns a map of rendered templates belonging to the // dependent packs of the parent template. The map key represents the path and // file name of the template. -func (r *Rendered) DependentRenders() map[string]string { return r.dependentRenders } +func (r *Rendered) DependentRenders() map[string]string { return r.dependencyRenders } // LenDependentRenders returns the number of dependent rendered templates that // are stored. -func (r *Rendered) LenDependentRenders() int { return len(r.dependentRenders) } +func (r *Rendered) LenDependentRenders() int { return len(r.dependencyRenders) } diff --git a/internal/pkg/testfixture/testfixture.go b/internal/pkg/testfixture/testfixture.go new file mode 100644 index 00000000..f7850412 --- /dev/null +++ b/internal/pkg/testfixture/testfixture.go @@ -0,0 +1,73 @@ +package testfixture + +import ( + "fmt" + "os/exec" + "path" + "strings" + "testing" + + "github.com/shoenig/test/must" +) + +var RelFixtureDir = "fixtures" + +// AbsPath returns the absolute path to a fixture inside the FixtureDir +func AbsPath(t *testing.T, fixtureName string) string { + t.Helper() + return path.Join(getRepoRoot(t), RelFixtureDir, fixtureName) +} + +// Clone creates a test TempDir, copies the given Pack into it, and returns the +// absolute path to the copy. +func Clone(t *testing.T, fPath string) (dest string) { + t.Helper() + parts := strings.Split(fPath, "/") + + td := t.TempDir() + p := AbsPath(t, fPath) + cmd := exec.Command("cp", "-R", p, td) + out, err := cmd.CombinedOutput() + must.NoError(t, err, must.Sprintf("output: %s\n err: %v", out, err)) + return path.Join(td, parts[len(parts)-1]) +} + +// getRepoRoot uses git rev-parse to locate the top folder in the git repo for +// locating the fixtures folder +func getRepoRoot(t *testing.T) string { + repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + must.NoError(t, err, must.Sprintf("output: %s\n err: %v", repoRoot, err)) + return strings.TrimSpace(string(repoRoot)) +} + +// MustAbsPath returns the absolute path to a fixture inside the FixtureDir +func MustAbsPath(fixtureName string) string { + // mustGetRepoRoot will panic on error, so this becomes a Must func too + return path.Join(mustGetRepoRoot(), RelFixtureDir, fixtureName) +} + +// Clone creates a test TempDir, copies the given Pack into it, and returns the +// absolute path to the copy. +func MustClone(dst string, fPath string) (dest string) { + p := MustAbsPath(fPath) + + cmd := exec.Command("cp", "-R", p, dst) + out, err := cmd.CombinedOutput() + if err != nil { + panic(fmt.Sprintf("MustClone fatal error:\nerr:%v\nout: %s", err, out)) + } + + parts := strings.Split(fPath, "/") + return path.Join(dst, parts[len(parts)-1]) +} + +// mustGetRepoRoot uses git rev-parse to locate the top folder in the git repo for +// locating the fixtures folder. If there is an error running the command, it will +// panic. This function is used in cases where we have no access to *testing.T +func mustGetRepoRoot() string { + repoRoot, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + panic(err) + } + return strings.TrimSpace(string(repoRoot)) +} diff --git a/internal/pkg/varfile/ctyhelpers.go b/internal/pkg/varfile/ctyhelpers.go new file mode 100644 index 00000000..c7fe79fd --- /dev/null +++ b/internal/pkg/varfile/ctyhelpers.go @@ -0,0 +1,37 @@ +package varfile + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" +) + +func traversalToName(t hcl.Traversal) []string { + acc := make([]string, len(t)) // make an accumulator + travToNameR(t, 0, &acc) + return acc +} + +func travToNameR(t hcl.Traversal, cur int, acc *[]string) { + if len(t) == 0 { // base case for the recursion + return + } + (*acc)[cur] = getStepName(t[0]) + travToNameR(t[1:], cur+1, acc) +} + +func getStepName(t any) string { + if _, ok := t.(hcl.Traverser); !ok { + panic(fmt.Sprintf("can't getStepName for non hcl.Traverser type %T", t)) + } + switch tt := t.(type) { + case hcl.TraverseRoot: + return tt.Name + case hcl.TraverseAttr: + return tt.Name + case hcl.TraverseIndex: + return tt.Key.AsString() + default: + panic(fmt.Sprintf("can't getStepName for hcl.Traverser type %T", tt)) + } +} diff --git a/internal/pkg/varfile/fixture/constants.go b/internal/pkg/varfile/fixture/constants.go new file mode 100644 index 00000000..a775f74a --- /dev/null +++ b/internal/pkg/varfile/fixture/constants.go @@ -0,0 +1,47 @@ +package fixture + +import ( + "github.com/hashicorp/nomad-pack/sdk/pack" +) + +const GoodConfigfileHCL = `# variable answers +simple_raw_exec.child1.username="foo" +simple_raw_exec.child1.password="bar" +simple_raw_exec.rootuser="admin" +` + +const GoodConfigfileJSON = `{ + "simple_raw_exec.child1.username": "foo", + "simple_raw_exec.child1.password": "bar", + "simple_raw_exec.rootuser": "admin" +}` + +const BadMissingEqualOneLine = `mypack.foo "bar"` +const BadMissingEqualSecondLine = `mypack.foo = "bar" +bad value` +const BadMissingEqualInternalLine = `mypack.foo = "bar" +bad value +mypack.bar = "baz"` + +const BadJSONMissingStartBrace = `"mypack.foo": "bar" }` +const BadJSONMissingEndBrace = `{ "mypack.foo": "bar"` +const BadJSONMissingComma = `{ "mypack.foo": "bar" "mypack.bar": "baz" }` +const BadJSONMissingQuote = `{ "mypack.foo": "bar", mypack.bar": "baz" }` +const BadJSONMissingColon = `{ "mypack.foo": "bar", mypack.bar" "baz" }` +const JSONEmpty = "" +const JSONEmptyObject = "{}" + +var JSONFiles = map[pack.ID][]*pack.File{ + "myPack": { + { + Name: "tc1.json", + Content: []byte(BadJSONMissingStartBrace), + Path: "/tmp/tc1.json", + }, + { + Name: "tc2.json", + Content: []byte(BadJSONMissingEndBrace), + Path: "/tmp/tc2.json", + }, + }, +} diff --git a/internal/pkg/varfile/varfile.go b/internal/pkg/varfile/varfile.go new file mode 100644 index 00000000..46dc6c5a --- /dev/null +++ b/internal/pkg/varfile/varfile.go @@ -0,0 +1,329 @@ +package varfile + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/json" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" +) + +func DecodeVariableOverrides(files []*pack.File) DecodeResult { + decodeResult := DecodeResult{} + for _, file := range files { + fileDecodeResult := DecodeResult{ + Overrides: make(variables.Overrides), + } + fileDecodeResult.HCLFiles, fileDecodeResult.Diags = Decode(file.Name, file.Content, nil, &fileDecodeResult.Overrides) + decodeResult.Merge(fileDecodeResult) + } + return decodeResult +} + +// DecodeResult is returned by the +type DecodeResult struct { + Overrides variables.Overrides + Diags hcl.Diagnostics + HCLFiles map[string]*hcl.File +} + +// Merge combines two DecodeResults. Any nil Overrides are discarded. +func (d *DecodeResult) Merge(in DecodeResult) { + // If the incoming DecodeResult contains diags, add them to our Diags + // collection + d.Diags = d.Diags.Extend(in.Diags) + + // For each of the incoming DecodeResults Overrides, which is a PackID-keyed + // map of Pack Variables and their Values. + for packID, packOverrides := range in.Overrides { + + // Traverse the incoming pack's overrides + for _, inOverride := range packOverrides { + + if inOverride == nil { + continue + } + + // If any existing values in the destination pack's slice conflict with + // the current value, they will be stored here since otherwise they'd be + // out of scope + var match *variables.Override + + // exactMatch is used to signal that the two Override values have the + // same pointer address so that the code can special case that. + var exactMatch bool + + if slices.ContainsFunc(d.Overrides[packID], func(e *variables.Override) bool { + // If either one of the values is nil, then it isn't a match for this + // purpose. + if inOverride == nil || d.Overrides[packID] == nil { + return false + } + + // Set the sentinel bool in the unlikely case that there is an + // exact match on the pointer address, so that the later code + // can choose how to deal with it. + if exactMatch = e == inOverride; exactMatch { + return true + } + + // If an override has the same name and path it is a conflict + if e.Name == inOverride.Name && e.Path == inOverride.Path { + // Retain the conflicting existing value's pointer so it can be used + // in generating an error + match = e + return true + } + + return false + }) { + + // Since it's an exact match, the destination already contains the + // incoming override; skip it + if exactMatch { + continue + } + + // Since there's an existing override in the destination, add an HCL + // diagnostic to the destination + d.Diags = d.Diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Duplicate definition", + Detail: fmt.Sprintf("The variable %s can not be redefined. Existing definition found at %s ", inOverride.Name, match.Range), + Subject: &inOverride.Range, + }) + + // Continue in order to collect more diagnostics if they happen + continue + } + + // Add the Override to the destination Overrides struct + d.Overrides[packID] = append(d.Overrides[packID], inOverride) + } + } + if d.HCLFiles == nil && in.HCLFiles != nil { + d.HCLFiles = in.HCLFiles + } + + for n, f := range in.HCLFiles { + d.HCLFiles[n] = f + } +} + +// Decode parses, decodes, and evaluates expressions in the given HCL source +// code, in a single step. +func Decode(filename string, src []byte, ctx *hcl.EvalContext, target *variables.Overrides) (map[string]*hcl.File, hcl.Diagnostics) { + fm, diags := decode(filename, src, ctx, target) + var fd = fixableDiags(diags) + + fm.Fixup() // the hcl.File that we will return to the diagnostic printer will have our modifications + fd.Fixup() // The Ranges in the diags will all be based on our modified byte slice + + return fm, hcl.Diagnostics(fd) +} + +// Decode parses, decodes, and evaluates expressions in the given HCL source +// code, in a single step. +func decode(filename string, src []byte, ctx *hcl.EvalContext, target *variables.Overrides) (diagFileMap, hcl.Diagnostics) { + var file *hcl.File + var diags hcl.Diagnostics + + // fm is a map of HCL filename to *hcl.File so the caller can use a + // hcl.DiagnosticWriter and pretty print the errors with contextual + // information. + var fm = make(diagFileMap) + + // Select the appropriate parser based on the file's extension + switch suffix := strings.ToLower(filepath.Ext(filename)); suffix { + case ".hcl": + wrapHCLBytes(&src) + file, diags = hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1}) + fm[filename] = file + case ".json": + wrapJSONBytes(&src) + file, diags = json.Parse(src, filename) + fm[filename] = file + default: + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported file format", + Detail: fmt.Sprintf("Cannot read from %s: unrecognized file format suffix %q.", filename, suffix), + }) + } + + // Any diags set at this point aren't recoverable, so return them. + if diags.HasErrors() { + return fm, diags + } + + // Because we wrap the user provided values in a HCL map or JSON object both + // named `v`, we can use JustAttributes to parse the configuration, and obtain + // the user-supplied content getting the `v` attribute's Expression and using + // ExprMap to convert it into a []hcl.KeyValuePair + attrs, diags := file.Body.JustAttributes() + expr := attrs["v"].Expr + em, emDiags := hcl.ExprMap(expr) + + // Any diags set at this point still aren't partially usable, so return them. + if emDiags.HasErrors() { + return fm, diags.Extend(emDiags) + } + + vals := make([]*variables.Override, 0, len(em)) + for _, kv := range em { + + // Read the value. If that generates diags, collect them, and stop + // processing this item. + value, vDiags := kv.Value.Value(nil) + if vDiags.HasErrors() { + diags = diags.Extend(vDiags) + continue + } + + // `steps` are the path components, so named because in the HCL case, they + // are gleaned by waking through the steps in a traversal and getting their + // names. + var steps []string + + // Read the key. Start by seeing if there are HCL variables in it, which is + // what a dotted identifier looks like in the HCL syntax case. + keyVars := kv.Key.Variables() + switch len(keyVars) { + case 0: + // In the JSON case, there's no tricks necessary to get the key, but + // we split on . to make the steps slice so the cases converge nicely. + + // I don't think there'd be a way to get diags in this case, so let's + // ignore them for the time being. + k, _ := kv.Key.Value(nil) + steps = strings.Split(k.AsString(), ".") + + case 1: + // In the HCL case, we have to read the traversal to get the path parts. + steps = traversalToName(keyVars[0]) + + default: + // If this happens, then there's something really wrong with my algorithm + // This might have to be relaxed after testing it a while. + panic("Too many variables in key name") + } + + // Create a range that represents the sum of the key and value ranges. + oRange := hcl.RangeBetween(kv.Key.Range(), kv.Value.Range()) + fixupRange(&oRange) + + val := variables.Override{ + Name: variables.ID(steps[len(steps)-1]), + Path: pack.ID(strings.Join(steps[0:len(steps)-1], ".")), + Value: value, + Type: value.Type(), + Range: oRange, + } + vals = append(vals, &val) + } + + if len(vals) > 0 { + (*target)[pack.ID(filename)] = vals + } + return fm, diags +} + +// wrapHCLBytes takes simple key-value structured HCL and converts them to HCL +// map syntax for parsing +func wrapHCLBytes(sp *[]byte) { + wrapBytes(sp, []byte("v = {\n"), []byte("\n}")) +} + +// wrapHCLBytes takes simple map structured JSON and converts it to HCL +// object syntax for parsing +func wrapJSONBytes(sp *[]byte) { + wrapBytes(sp, []byte(`{"v":`+"\n"), []byte("\n}")) +} + +// wrapBytes is a convenience function to make wrapping byte slices easier +// to read +func wrapBytes(b *[]byte, prefix, postfix []byte) { + *b = append(append(prefix, *b...), postfix...) +} + +// unwrapBytes reverses the changes made by wrapHCLBytes and wrapJSONBytes +func unwrapBytes(sp *[]byte) { + src := *sp + // Trim the first 6 and last 2 bytes (since we added those). + out := slices.Clip(src[6 : len(src)-2]) + *sp = out +} + +func fixupRange(r *hcl.Range) { + fixupPos(&r.Start) + fixupPos(&r.End) +} + +func fixupPos(p *hcl.Pos) { + + // Adjust the byte position to account for the map wrapper that we have to + // take back out + p.Byte -= 6 + + // Some ranges, especially the "Context" ones, might refer to the line we + // insert to cheat into parsing the value as a map. Setting it to the home + // position on line two works nicely with the subtraction we have to do in + // all the other cases. + if p.Line == 1 { + p.Byte = 0 // Step on the computed Byte val, because it will be negative + p.Line = 2 // Set to 2 since Line is always decremented by one + p.Column = 1 // The first column aligns with the zero byte. + } + + p.Line -= 1 +} + +// DiagExtraFixup is a custom type for the sentinel value stored in Extra that +// indicates whether or not the Ranges in the Diagnostic have been reset to be +// consistent with the user-supplied input. +type DiagExtraFixup struct{ Fixed bool } + +// markFixed adds the sentinel value to the passed Diagnostic +func markFixed(d *hcl.Diagnostic) { d.Extra = DiagExtraFixup{Fixed: true} } + +// The HCL DiagnosticsWriter can use a map of filenames to parsed HCL files to +// enrich the Diagnostic's output with actual file content. +type diagFileMap map[string]*hcl.File + +// Fixup removes the data added to the original inputs so that they print out +// as they were originally provided. +func (d *diagFileMap) Fixup() { + // We need to fix all of the byte arrays so that they have the original data + for _, f := range *d { + unwrapBytes(&f.Bytes) + } +} + +// fixableDiags are diagnostics that have additional funcs attached to true +// up the data stored in the Ranges before presenting them to the user. +type fixableDiags hcl.Diagnostics + +// Fixup adjusts the ranges of Diagnostics that have ranges different than they +// should because of the manipulation of the input data. For example, the Pack +// v2 variable override files are modified dynamically to transform them into +// HCL2 maps, which allows up to cheat and use dotted identifiers. However, this +// modification causes the Ranges to be incorrect on any hcl.Diagnostic based on +// them. Fixup calculates where the ranges would have originally referred to. +func (f *fixableDiags) Fixup() { + for _, diag := range *f { + if diag.Extra == nil { + if diag.Subject != nil { + fixupRange(diag.Subject) + } + if diag.Context != nil { + fixupRange(diag.Context) + } + markFixed(diag) + } + } +} diff --git a/internal/pkg/varfile/varfile_pack_test.go b/internal/pkg/varfile/varfile_pack_test.go new file mode 100644 index 00000000..bed845a3 --- /dev/null +++ b/internal/pkg/varfile/varfile_pack_test.go @@ -0,0 +1,30 @@ +package varfile_test + +import ( + "os" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/varfile" + "github.com/hashicorp/nomad-pack/internal/pkg/varfile/fixture" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/shoenig/test/must" +) + +func TestVarfile_ProcessPackVarfiles(t *testing.T) { + ovrds := make(variables.Overrides) + fm, d := varfile.Decode("foo.hcl", []byte(`foo="bar"`), nil, &ovrds) + if d.HasErrors() { + dw := hcl.NewDiagnosticTextWriter(os.Stderr, fm, 40, false) + t.Log(dw.WriteDiagnostics(d)) + t.FailNow() + } + must.Len[*variables.Override](t, 1, ovrds["foo.hcl"]) + must.Eq(t, "bar", ovrds["foo.hcl"][0].Value.AsString()) +} + +func TestVarfile_DecodeVariableOverrides(t *testing.T) { + dr := varfile.DecodeVariableOverrides(fixture.JSONFiles["myPack"]) + dw := hcl.NewDiagnosticTextWriter(os.Stderr, dr.HCLFiles, 80, false) + dw.WriteDiagnostics(dr.Diags) +} diff --git a/internal/pkg/varfile/varfile_test.go b/internal/pkg/varfile/varfile_test.go new file mode 100644 index 00000000..a7856f98 --- /dev/null +++ b/internal/pkg/varfile/varfile_test.go @@ -0,0 +1,216 @@ +package varfile + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/shoenig/test/must" + "github.com/zclconf/go-cty/cty" +) + +func TestVarfile_DecodeHCL(t *testing.T) { + type exp struct { + dLen int + diags hcl.Diagnostics + oLen int + oMap variables.Overrides + } + testCases := []struct { + name string + src []byte + exp exp + }{ + { + name: "empty", + src: []byte{}, + exp: exp{}, + }, + { + name: "comment only", + src: []byte("# just a comment"), + exp: exp{}, + }, + { + name: "single override", + src: []byte(`mypack.foo = "bar"`), + exp: exp{ + dLen: 0, + oLen: 1, + oMap: variables.Overrides{ + "embedded.hcl": []*variables.Override{ + { + Name: "foo", + Path: "mypack", + Type: cty.String, + Value: cty.StringVal("bar"), + Range: hcl.Range{ + Filename: "embedded.hcl", + Start: hcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: hcl.Pos{Line: 1, Column: 19, Byte: 18}, + }, + }, + }, + }, + }, + }, + { + name: "missing equal", + src: []byte(`mypack.foo "bar"`), + exp: exp{ + dLen: 1, + diags: hcl.Diagnostics{ + &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: `Missing key/value separator`, + Detail: `Expected an equals sign ("=") to mark the beginning of the attribute value.`, + Subject: &hcl.Range{ + Filename: "embedded.hcl", + Start: hcl.Pos{Line: 1, Column: 12, Byte: 11}, + End: hcl.Pos{Line: 1, Column: 13, Byte: 12}, + }, + Context: &hcl.Range{ + Filename: "embedded.hcl", + Start: hcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: hcl.Pos{Line: 1, Column: 13, Byte: 12}, + }, + EvalContext: nil, + Extra: DiagExtraFixup{Fixed: true}, + }, + }, + }, + }, + { + name: "error only", + src: []byte("boom"), + exp: exp{ + dLen: 1, + oLen: 0, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc := tc + om := make(variables.Overrides) + _, diags := Decode("embedded.hcl", tc.src, nil, &om) + must.Len(t, tc.exp.dLen, diags, must.Sprintf("slice values: %v", diags)) + + if len(tc.exp.diags) > 0 { + for i, o := range diags { + e := tc.exp.diags[i] + must.Eq(t, e, o) + } + } + // There should always be a single element map for these tests having the + // filename as the key + switch tc.exp.oLen { + case 0: + must.MapLen[variables.Overrides](t, 0, om, must.Sprintf("map contents: %v", spew.Sdump(om))) + default: + must.MapLen[variables.Overrides](t, 1, om, must.Sprintf("map contents: %v", spew.Sdump(om))) + must.MapContainsKey[variables.Overrides](t, om, "embedded.hcl") + } + + if len(tc.exp.oMap) > 0 { + // Extract the expect slice + eSlice := tc.exp.oMap["embedded.hcl"] + // Extract the slice + oSlice := om["embedded.hcl"] + must.SliceLen[*variables.Override](t, tc.exp.oLen, oSlice, must.Sprintf("slice values: %v", oSlice)) + + for i, o := range oSlice { + e := eSlice[i] + must.True(t, e.Equal(o)) + } + } + }) + } +} + +func TestVarfile_DecodeResult_Merge(t *testing.T) { + d1 := DecodeResult{ + Overrides: variables.Overrides{ + "p1": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + t.Run("errors when redefined", func(t *testing.T) { + dr := DecodeResult{ + Overrides: variables.Overrides{ + "p1": []*variables.Override{{Name: "o1"}}, + }, + } + + dr.Merge(d1) + must.True(t, dr.Diags.HasErrors()) + must.ErrorContains(t, dr.Diags, "variable o1 can not be redefined") + }) +} + +func TestVarFile_Merge_Good(t *testing.T) { + d1 := DecodeResult{ + Overrides: variables.Overrides{ + "p1": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + + t.Run("succeeds for", func(t *testing.T) { + t.Run("okay for variables of same name in different pack", func(t *testing.T) { + dr := DecodeResult{ + Overrides: variables.Overrides{ + "p2": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + dr.Merge(d1) + must.False(t, dr.Diags.HasErrors()) + must.Len[*variables.Override](t, 2, dr.Overrides["p1"]) + must.Len[*variables.Override](t, 2, dr.Overrides["p2"]) + }) + + t.Run("okay for repeated pointers to same override", func(t *testing.T) { + dr := DecodeResult{ + Overrides: variables.Overrides{ + "p2": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + dr2 := dr + dr2.Merge(dr) + must.False(t, dr.Diags.HasErrors()) + must.Len[*variables.Override](t, 2, dr.Overrides["p2"]) + }) + + t.Run("for nil overrides", func(t *testing.T) { + dr := DecodeResult{ + Overrides: variables.Overrides{ + "p2": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + d2 := DecodeResult{ + Overrides: variables.Overrides{}, + } + dr.Merge(d2) + must.False(t, dr.Diags.HasErrors()) + must.Len[*variables.Override](t, 2, dr.Overrides["p2"]) + }) + + t.Run("for nil override pointer", func(t *testing.T) { + + d1 := DecodeResult{ + Overrides: variables.Overrides{ + "p1": []*variables.Override{{Name: "o1"}, {Name: "o2"}}, + }, + } + var nilPtr *variables.Override + d := DecodeResult{ + Overrides: variables.Overrides{ + "p1": []*variables.Override{nilPtr}, + }, + } + + d1.Merge(d) + must.False(t, d1.Diags.HasErrors()) + must.Len(t, 2, d1.Overrides["p1"]) + }) + }) +} diff --git a/internal/pkg/variable/cli.go b/internal/pkg/variable/cli.go deleted file mode 100644 index 0641a973..00000000 --- a/internal/pkg/variable/cli.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "fmt" - "os" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" -) - -func (p *Parser) parseEnvVariable(name string, rawVal string) hcl.Diagnostics { - // Split the name to see if we have a namespace CLI variable for a child - // pack and set the default packVarName. - splitName := strings.SplitN(name, ".", 2) - packVarName := []string{p.cfg.ParentName, name} - - switch len(splitName) { - case 1: - // Fallthrough, nothing to do or see. - case 2: - // We are dealing with a namespaced variable. Overwrite the preset - // values of packVarName. - packVarName[0] = splitName[0] - packVarName[1] = splitName[1] - default: - // We cannot handle a splitName where the variable includes more than - // one separator. - return hcl.Diagnostics{ - { - Severity: hcl.DiagError, - Summary: fmt.Sprintf("Invalid %s option", strings.TrimRight(VarEnvPrefix, "_")), - Detail: fmt.Sprintf("The given environment variable %s%s=%s is not correctly specified. The variable name must not have more than one dot `.` separator.", VarEnvPrefix, name, rawVal), - }, - } - } - - // Generate a filename based on the CLI var, so we have some context for any - // HCL diagnostics. - fakeRange := hcl.Range{Filename: fmt.Sprintf("", name)} - - // If the variable has not been configured in the root then ignore it. This - // is a departure from the way in which flags and var-files are handled. - // The environment might contain NOMAD_PACK_VAR variables used for other - // packs that might be run on the same system but are not used with this - // particular pack. - existing, exists := p.rootVars[packVarName[0]][packVarName[1]] - if !exists { - return nil - } - - expr, diags := expressionFromVariableDefinition(fakeRange.Filename, rawVal, existing.Type) - if diags.HasErrors() { - return diags - } - - val, diags := expr.Value(nil) - if diags.HasErrors() { - return diags - } - - // If our stored type isn't cty.NilType then attempt to covert the override - // variable, so we know they are compatible. - if existing.Type != cty.NilType { - var err *hcl.Diagnostic - val, err = convertValUsingType(val, existing.Type, expr.Range().Ptr()) - if err != nil { - return hcl.Diagnostics{err} - } - } - - // We have a verified override variable. - v := Variable{ - Name: packVarName[1], - Type: val.Type(), - Value: val, - DeclRange: fakeRange, - } - p.envOverrideVars[packVarName[0]] = append(p.envOverrideVars[packVarName[0]], &v) - - return nil -} - -func (p *Parser) parseCLIVariable(name string, rawVal string) hcl.Diagnostics { - // Split the name to see if we have a namespace CLI variable for a child - // pack and set the default packVarName. - splitName := strings.SplitN(name, ".", 2) - packVarName := []string{p.cfg.ParentName, name} - - switch len(splitName) { - case 1: - // Fallthrough, nothing to do or see. - case 2: - // We are dealing with a namespaced variable. Overwrite the preset - // values of packVarName. - packVarName[0] = splitName[0] - packVarName[1] = splitName[1] - default: - // We cannot handle a splitName where the variable includes more than - // one separator. - return hcl.Diagnostics{ - { - Severity: hcl.DiagError, - Summary: "Invalid -var option", - Detail: fmt.Sprintf("The given -var option %s=%s is not correctly specified. The variable name must not have more than one dot `.` separator.", name, rawVal), - }, - } - } - - // Generate a filename based on the CLI var, so we have some context for any - // HCL diagnostics. - fakeRange := hcl.Range{Filename: fmt.Sprintf("", name)} - - // If the variable has not been configured in the root then exit. This is a - // standard requirement, especially because we would be unable to ensure a - // consistent type. - existing, exists := p.rootVars[packVarName[0]][packVarName[1]] - if !exists { - return hcl.Diagnostics{diagnosticMissingRootVar(name, &fakeRange)} - } - - expr, diags := expressionFromVariableDefinition(fakeRange.Filename, rawVal, existing.Type) - if diags.HasErrors() { - return diags - } - - val, diags := expr.Value(nil) - if diags.HasErrors() { - return diags - } - - // If our stored type isn't cty.NilType then attempt to covert the override - // variable, so we know they are compatible. - if existing.Type != cty.NilType { - var err *hcl.Diagnostic - val, err = convertValUsingType(val, existing.Type, expr.Range().Ptr()) - if err != nil { - return hcl.Diagnostics{err} - } - } - - // We have a verified override variable. - v := Variable{ - Name: packVarName[1], - Type: val.Type(), - Value: val, - DeclRange: fakeRange, - } - p.cliOverrideVars[packVarName[0]] = append(p.cliOverrideVars[packVarName[0]], &v) - - return nil -} - -// expressionFromVariableDefinition attempts to convert the string HCL -// expression to a hydrated hclsyntax.Expression. -func expressionFromVariableDefinition(file, val string, varType cty.Type) (hclsyntax.Expression, hcl.Diagnostics) { - switch varType { - case cty.String, cty.Number, cty.NilType: - return &hclsyntax.LiteralValueExpr{Val: cty.StringVal(val)}, nil - default: - return hclsyntax.ParseExpression([]byte(val), file, hcl.Pos{Line: 1, Column: 1}) - } -} - -func GetVarsFromEnv() map[string]string { - out := make(map[string]string) - - for _, raw := range os.Environ() { - if !strings.HasPrefix(raw, VarEnvPrefix) { - continue - } - raw = raw[len(VarEnvPrefix):] // trim the prefix - - eq := strings.Index(raw, "=") - if eq == -1 { - // Seems invalid, so we'll ignore it. - continue - } - - name := raw[:eq] - value := raw[eq+1:] - out[name] = value - } - - return out -} diff --git a/internal/pkg/variable/decode.go b/internal/pkg/variable/decode.go deleted file mode 100644 index aa03e72b..00000000 --- a/internal/pkg/variable/decode.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "fmt" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/ext/typeexpr" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" -) - -func decodeVariableBlock(block *hcl.Block) (*Variable, hcl.Diagnostics) { - - content, diags := block.Body.Content(variableBlockSchema) - if content == nil { - return nil, diags - } - - if diags == nil { - diags = hcl.Diagnostics{} - } - - v := &Variable{ - Name: block.Labels[0], - DeclRange: block.DefRange, - } - - // Ensure the variable name is valid. If this isn't checked it will cause - // problems in future use. - if !hclsyntax.ValidIdentifier(v.Name) { - diags = diags.Append(diagnosticInvalidVariableName(v.DeclRange.Ptr())) - } - - // A variable doesn't need to declare a description. If it does, process - // this and store it, along with any processing errors. - if attr, exists := content.Attributes[variableAttributeDescription]; exists { - val, descDiags := attr.Expr.Value(nil) - diags = safeDiagnosticsExtend(diags, descDiags) - - if val.Type() == cty.String { - v.hasDescription = true - v.Description = val.AsString() - } else { - diags = safeDiagnosticsAppend(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid type for description", - Detail: fmt.Sprintf("The description attribute is expected to be of type string, got %s", - val.Type().FriendlyName()), - Subject: attr.Range.Ptr(), - }) - } - } - - // A variable doesn't need to declare a type. If it does, process this and - // store it, along with any processing errors. - if attr, exists := content.Attributes[variableAttributeType]; exists { - ty, tyDiags := typeexpr.Type(attr.Expr) - diags = safeDiagnosticsExtend(diags, tyDiags) - v.hasType = true - v.Type = ty - } - - // A variable doesn't need to declare a default. If it does, process this - // and store it, along with any processing errors. - if attr, exists := content.Attributes[variableAttributeDefault]; exists { - val, valDiags := attr.Expr.Value(nil) - diags = safeDiagnosticsExtend(diags, valDiags) - - // If the found type isn't cty.NilType, then attempt to covert the - // default variable, so we know they are compatible. - if v.Type != cty.NilType { - var err *hcl.Diagnostic - val, err = convertValUsingType(val, v.Type, attr.Expr.Range().Ptr()) - diags = safeDiagnosticsAppend(diags, err) - } - v.hasDefault = true - v.Default = val - v.Value = val - } - - return v, diags -} - -// convertValUsingType is a wrapper around convert.Convert. -func convertValUsingType(val cty.Value, typ cty.Type, sub *hcl.Range) (cty.Value, *hcl.Diagnostic) { - newVal, err := convert.Convert(val, typ) - if err != nil { - return cty.DynamicVal, diagnosticInvalidValueForType(err, sub) - } - return newVal, nil -} diff --git a/internal/pkg/variable/decoder/decode.go b/internal/pkg/variable/decoder/decode.go new file mode 100644 index 00000000..b94df899 --- /dev/null +++ b/internal/pkg/variable/decoder/decode.go @@ -0,0 +1,96 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package decoder + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/typeexpr" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/internal/hclhelp" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/schema" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/zclconf/go-cty/cty" +) + +// DecodeVariableBlock parses a variable definition into a variable. When the +// provided block or its Body is nil, the function returns (nil, nil) +func DecodeVariableBlock(block *hcl.Block) (*variables.Variable, hcl.Diagnostics) { + if block == nil || block.Body == nil { + return nil, hcl.Diagnostics{} + } + + // If block and Body is non-nil, then the block is ready to be parsed + content, diags := block.Body.Content(schema.VariableBlockSchema) + if content == nil { + return nil, diags + } + + if diags == nil { + diags = hcl.Diagnostics{} + } + + v := &variables.Variable{ + Name: variables.ID(block.Labels[0]), + DeclRange: block.DefRange, + } + + // Ensure the variable name is valid. If this isn't checked it will cause + // problems in future use. + if !hclsyntax.ValidIdentifier(v.Name.String()) { + diags = diags.Append(packdiags.DiagInvalidVariableName(v.DeclRange.Ptr())) + } + + // A variable doesn't need to declare a description. If it does, process + // this and store it, along with any processing errors. + if attr, exists := content.Attributes[schema.VariableAttributeDescription]; exists { + val, descDiags := attr.Expr.Value(nil) + diags = packdiags.SafeDiagnosticsExtend(diags, descDiags) + + if val.Type() == cty.String { + v.SetDescription(val.AsString()) + } else { + diags = packdiags.SafeDiagnosticsAppend(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid type for description", + Detail: fmt.Sprintf("The description attribute is expected to be of type string, got %s", + val.Type().FriendlyName()), + Subject: attr.Range.Ptr(), + }) + } + } + + // A variable doesn't need to declare a type. If it does, process this and + // store it, along with any processing errors. + if attr, exists := content.Attributes[schema.VariableAttributeType]; exists { + ty, tyDiags := typeexpr.Type(attr.Expr) + diags = packdiags.SafeDiagnosticsExtend(diags, tyDiags) + v.SetType(ty) + } + + // A variable doesn't need to declare a default. If it does, process this + // and store it, along with any processing errors. + if attr, exists := content.Attributes[schema.VariableAttributeDefault]; exists { + val, valDiags := attr.Expr.Value(nil) + diags = packdiags.SafeDiagnosticsExtend(diags, valDiags) + + // If the found type isn't cty.NilType, then attempt to covert the + // default variable, so we know they are compatible. + if v.Type != cty.NilType { + var err *hcl.Diagnostic + val, err = hclhelp.ConvertValUsingType(val, v.Type, attr.Expr.Range().Ptr()) + diags = packdiags.SafeDiagnosticsAppend(diags, err) + } + v.SetDefault(val) + v.Value = val + } + + if diags.HasErrors() { + return nil, diags + } + + return v, diags +} diff --git a/internal/pkg/variable/decoder/decode_test.go b/internal/pkg/variable/decoder/decode_test.go new file mode 100644 index 00000000..605a0acf --- /dev/null +++ b/internal/pkg/variable/decoder/decode_test.go @@ -0,0 +1,178 @@ +package decoder + +import ( + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/schema" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/hashicorp/nomad/ci" + "github.com/shoenig/test/must" + "github.com/zclconf/go-cty/cty" +) + +func TestDecoder_DecodeVariableBlock(t *testing.T) { + ci.Parallel(t) + + testCases := []struct { + name string + input *hcl.Block + expectOut *variables.Variable + expectDiags hcl.Diagnostics + shouldErr bool + }{ + { + name: "passes/on nil block", + input: &hcl.Block{}, + expectOut: nil, + expectDiags: hcl.Diagnostics{}, + }, + { + name: "passes/on minimal block", + input: testGetHCLBlock(t, testLoadPackFile(t, []byte(goodMinimalVariableHCL))), + expectOut: func() *variables.Variable { + out := variables.Variable{ + Name: "good", + DeclRange: hcl.Range{ + Filename: "/fake/test/path", + Start: hcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: hcl.Pos{Line: 1, Column: 18, Byte: 17}, + }, + } + return &out + }(), + expectDiags: hcl.Diagnostics{}, + }, + { + name: "passes/on good block", + input: testGetHCLBlock(t, testLoadPackFile(t, []byte(goodCompleteVariableHCL))), + expectOut: func() *variables.Variable { + out := variables.Variable{ + Name: "example", + DeclRange: hcl.Range{ + Filename: "/fake/test/path", + Start: hcl.Pos{Line: 2, Column: 1, Byte: 1}, + End: hcl.Pos{Line: 2, Column: 19, Byte: 19}, + }, + } + out.SetDefault(cty.StringVal("default")) + out.SetDescription("an example variable") + out.SetType(cty.String) + out.Value = cty.StringVal("default") + return &out + }(), + expectDiags: hcl.Diagnostics{}, + }, + { + name: "fails/on bad content", + input: testGetHCLBlock(t, testLoadPackFile(t, []byte(badContent))), + expectOut: nil, + expectDiags: hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported block type", + Detail: "Blocks of type \"bad\" are not expected here.", + Subject: &hcl.Range{ + Filename: "/fake/test/path", + Start: hcl.Pos{Line: 2, Column: 2, Byte: 22}, + End: hcl.Pos{Line: 2, Column: 5, Byte: 25}, + }, + }}, + }, + { + name: "fails/on bad name", + input: testGetHCLBlock(t, testLoadPackFile(t, []byte(badNameText))), + expectOut: nil, + expectDiags: hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid variable name", + Detail: "Name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.", + Subject: &hcl.Range{ + Filename: "/fake/test/path", + Start: hcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: hcl.Pos{Line: 1, Column: 17, Byte: 16}, + }, + }}, + }, + { + name: "fails/on bad description type", + input: testGetHCLBlock(t, testLoadPackFile(t, []byte(badDescriptionType))), + expectOut: nil, + expectDiags: hcl.Diagnostics{&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid type for description", + Detail: "The description attribute is expected to be of type string, got bool", + Subject: &hcl.Range{ + Filename: "/fake/test/path", + Start: hcl.Pos{Line: 2, Column: 2, Byte: 18}, + End: hcl.Pos{Line: 2, Column: 20, Byte: 36}, + }, + }}, + }, + } + for _, tc := range testCases { + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) + out, diags := DecodeVariableBlock(tc.input) + must.Eq(t, tc.expectOut, out) + if tc.expectDiags != nil { + spew.Config.DisableMethods = true + must.SliceContainsAll(t, tc.expectDiags, diags, must.Sprint(spew.Sprint(tc.expectDiags)), must.Sprint(spew.Sprintf("%v", diags))) + } + }) + } +} + +const goodMinimalVariableHCL = `variable "good" {}` + +const goodCompleteVariableHCL = `variable "example" { + type = string + default = "default" + description = "an example variable" +}` + +const badContent = `variable "example" { + bad {} +}` + +const badNameText = `variable "!bad!" {}` + +const badDescriptionType = `variable "bad" { + description = true +}` + +// loadPackFile takes a pack.File and parses this using a hclparse.Parser. The +// file can be either HCL and JSON format. +func testLoadPackFile(t *testing.T, b []byte) hcl.Body { + t.Helper() + + var ( + hclFile *hcl.File + diags hcl.Diagnostics + ) + + hclParser := hclparse.NewParser() + hclFile, diags = hclParser.ParseHCL(b, "/fake/test/path") + + must.Len(t, 0, diags, must.Sprint(diags.Error())) + + // If the returned file or body is nil, then we'll return a non-nil empty + // body, so we'll meet our contract that nil means an error reading the + // file. + if hclFile == nil || hclFile.Body == nil { + return hcl.EmptyBody() + } + + must.Len(t, 0, diags, must.Sprint(diags.Error())) + return hclFile.Body +} + +func testGetHCLBlock(t *testing.T, in hcl.Body) *hcl.Block { + t.Helper() + b, diags := in.Content(schema.VariableFileSchema) + must.Len(t, 0, diags, must.Sprint(diags.Error())) + must.True(t, len(b.Blocks) >= 1) + return b.Blocks[0] +} diff --git a/internal/pkg/variable/envloader/envloader.go b/internal/pkg/variable/envloader/envloader.go new file mode 100644 index 00000000..7c16b038 --- /dev/null +++ b/internal/pkg/variable/envloader/envloader.go @@ -0,0 +1,39 @@ +package envloader + +import ( + "os" + "strings" +) + +const DefaultPrefix = "NOMAD_PACK_VAR_" + +type EnvLoader struct { + prefix string +} + +func New() *EnvLoader { + return &EnvLoader{prefix: DefaultPrefix} +} + +func (e *EnvLoader) GetVarsFromEnv() map[string]string { + if e.prefix == "" { + return getVarsFromEnv(DefaultPrefix) + } + return getVarsFromEnv(e.prefix) +} + +func getVarsFromEnv(prefix string) map[string]string { + out := make(map[string]string) + for _, raw := range os.Environ() { + switch { + case !strings.HasPrefix(raw, prefix): + continue + case !strings.Contains(raw, "="): + continue + default: + key, value, _ := strings.Cut(raw, "=") + out[strings.TrimPrefix(key, prefix)] = value + } + } + return out +} diff --git a/internal/pkg/variable/error.go b/internal/pkg/variable/error.go deleted file mode 100644 index b9d5a1e7..00000000 --- a/internal/pkg/variable/error.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "fmt" - - "github.com/hashicorp/hcl/v2" - - "github.com/hashicorp/nomad-pack/internal/pkg/helper" -) - -func safeDiagnosticsAppend(base hcl.Diagnostics, new *hcl.Diagnostic) hcl.Diagnostics { - if new != nil { - base = base.Append(new) - } - return base -} - -func safeDiagnosticsExtend(base, new hcl.Diagnostics) hcl.Diagnostics { - if new != nil && new.HasErrors() { - base = base.Extend(new) - } - return base -} - -func diagnosticMissingRootVar(name string, sub *hcl.Range) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Missing base variable declaration to override", - Detail: fmt.Sprintf(`There is no variable named %q. An override file can only override a variable that was already declared in a primary configuration file.`, name), - Subject: sub, - } -} - -func diagnosticInvalidDefaultValue(detail string, sub *hcl.Range) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid default value for variable", - Detail: detail, - Subject: sub, - } -} - -func diagnosticFailedToConvertCty(err error, sub *hcl.Range) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to convert Cty to interface", - Detail: helper.Title(err.Error()), - Subject: sub, - } -} -func diagnosticInvalidValueForType(err error, sub *hcl.Range) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid value for variable", - Detail: fmt.Sprintf("This variable value is not compatible with the variable's type constraint: %s.", err), - Subject: sub, - } -} - -func diagnosticInvalidVariableName(sub *hcl.Range) *hcl.Diagnostic { - return &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid variable name", - Detail: "Name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.", - Subject: sub, - } -} diff --git a/internal/pkg/variable/internal/hclhelp/hclhelp.go b/internal/pkg/variable/internal/hclhelp/hclhelp.go new file mode 100644 index 00000000..2a3f9d5e --- /dev/null +++ b/internal/pkg/variable/internal/hclhelp/hclhelp.go @@ -0,0 +1,29 @@ +package hclhelp + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +// expressionFromVariableDefinition attempts to convert the string HCL +// expression to a hydrated hclsyntax.Expression. +func ExpressionFromVariableDefinition(file, val string, varType cty.Type) (hclsyntax.Expression, hcl.Diagnostics) { + switch varType { + case cty.String, cty.Number, cty.NilType: + return &hclsyntax.LiteralValueExpr{Val: cty.StringVal(val)}, nil + default: + return hclsyntax.ParseExpression([]byte(val), file, hcl.Pos{Line: 1, Column: 1}) + } +} + +// convertValUsingType is a wrapper around convert.Convert. +func ConvertValUsingType(val cty.Value, typ cty.Type, sub *hcl.Range) (cty.Value, *hcl.Diagnostic) { + newVal, err := convert.Convert(val, typ) + if err != nil { + return cty.DynamicVal, packdiags.DiagInvalidValueForType(err, sub) + } + return newVal, nil +} diff --git a/internal/pkg/variable/parser.go b/internal/pkg/variable/parser.go deleted file mode 100644 index e28099ce..00000000 --- a/internal/pkg/variable/parser.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "errors" - "fmt" - "os" - "sort" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclparse" - "github.com/hashicorp/nomad-pack/sdk/pack" - "github.com/spf13/afero" - "github.com/zclconf/go-cty/cty" -) - -// Parser can parse, merge, and validate HCL variables from multiple different -// sources. -type Parser struct { - fs afero.Afero - cfg *ParserConfig - - // rootVars contains all the root variable declared by all parent and child - // packs that are being parsed. The first map is keyed by the pack name, - // the second is by the variable name. - rootVars map[string]map[string]*Variable - - // envOverrideVars, fileOverrideVars, cliOverrideVars are the override - // variables. The maps are keyed by the pack name they are associated to. - envOverrideVars map[string][]*Variable - fileOverrideVars map[string][]*Variable - cliOverrideVars map[string][]*Variable -} - -// ParserConfig contains details of the numerous sources of variables which -// should be parsed and merged according to the expected strategy. -type ParserConfig struct { - - // ParentName is the name representing the parent pack. - ParentName string - - // RootVariableFiles contains a map of root variable files, keyed by their - // pack name. - RootVariableFiles map[string]*pack.File - - // EnvOverrides are key=value variables and take the lowest precedence of - // all sources. If the same key is supplied twice, the last wins. - EnvOverrides map[string]string - - // FileOverrides is a list of files which contain variable overrides in the - // form key=value. The files will be stored before processing to ensure a - // consistent processing experience. Overrides here will replace any - // default root declarations. - FileOverrides []string - - // CLIOverrides are key=value variables and take the highest precedence of - // all sources. If the same key is supplied twice, the last wins. - CLIOverrides map[string]string - - // IgnoreMissingVars determines whether we error or not on variable overrides - // that don't have corresponding vars in the pack. - IgnoreMissingVars bool -} - -func NewParser(cfg *ParserConfig) (*Parser, error) { - - // Ensure the parent name is set, otherwise we can't parse correctly. - if cfg.ParentName == "" { - return nil, errors.New("variable parser config requires ParentName to be set") - } - - // Sort the file overrides to ensure variable merging is consistent on - // multiple passes. - sort.Strings(cfg.FileOverrides) - for _, file := range cfg.FileOverrides { - _, err := os.Stat(file) - if err != nil { - return nil, fmt.Errorf("variable file %q not found", file) - } - } - - return &Parser{ - fs: afero.Afero{ - Fs: afero.OsFs{}, - }, - cfg: cfg, - rootVars: make(map[string]map[string]*Variable), - envOverrideVars: make(map[string][]*Variable), - fileOverrideVars: make(map[string][]*Variable), - cliOverrideVars: make(map[string][]*Variable), - }, nil -} - -func (p *Parser) Parse() (*ParsedVariables, hcl.Diagnostics) { - - // Parse the root variables. If we encounter an error here, we are unable - // to reliably continue. - diags := p.parseRootFiles() - if diags.HasErrors() { - return nil, diags - } - - // Parse env, file, and CLI overrides. - for k, v := range p.cfg.EnvOverrides { - envOverrideDiags := p.parseEnvVariable(k, v) - diags = safeDiagnosticsExtend(diags, envOverrideDiags) - } - - for _, fileOverride := range p.cfg.FileOverrides { - fileOverrideDiags := p.parseOverridesFile(fileOverride) - diags = safeDiagnosticsExtend(diags, fileOverrideDiags) - } - - for k, v := range p.cfg.CLIOverrides { - cliOverrideDiags := p.parseCLIVariable(k, v) - diags = safeDiagnosticsExtend(diags, cliOverrideDiags) - } - - if diags.HasErrors() { - return nil, diags - } - - // Iterate all our override variables and merge these into our root - // variables with the CLI taking highest priority. - for _, override := range []map[string][]*Variable{p.envOverrideVars, p.fileOverrideVars, p.cliOverrideVars} { - for packName, variables := range override { - for _, v := range variables { - existing, exists := p.rootVars[packName][v.Name] - if !exists { - if !p.cfg.IgnoreMissingVars { - diags = diags.Append(diagnosticMissingRootVar(v.Name, v.DeclRange.Ptr())) - } - continue - } - if mergeDiags := existing.merge(v); mergeDiags.HasErrors() { - diags = diags.Extend(mergeDiags) - } - } - } - } - - return &ParsedVariables{Vars: p.rootVars}, diags -} - -func (p *Parser) loadOverrideFile(file string) (hcl.Body, hcl.Diagnostics) { - - src, err := p.fs.ReadFile(file) - // FIXME - Workaround for ending heredoc with no linefeed. - // Variables files shouldn't care about the extra linefeed, but jamming one - // in all the time feels bad. - src = append(src, "\n"...) - if err != nil { - return nil, hcl.Diagnostics{ - { - Severity: hcl.DiagError, - Summary: "Failed to read file", - Detail: fmt.Sprintf("The file %q could not be read.", file), - }, - } - } - - return p.loadPackFile(&pack.File{Path: file, Content: src}) -} - -// loadPackFile takes a pack.File and parses this using a hclparse.Parser. The -// file can be either HCL and JSON format. -func (p *Parser) loadPackFile(file *pack.File) (hcl.Body, hcl.Diagnostics) { - - var ( - hclFile *hcl.File - diags hcl.Diagnostics - ) - - // Instantiate a new parser each time. Using the same parser where variable - // names collide from different packs will cause problems. - hclParser := hclparse.NewParser() - - // Depending on the fix extension, use the correct HCL parser. - switch { - case strings.HasSuffix(file.Name, ".json"): - hclFile, diags = hclParser.ParseJSON(file.Content, file.Path) - default: - hclFile, diags = hclParser.ParseHCL(file.Content, file.Path) - } - - // If the returned file or body is nil, then we'll return a non-nil empty - // body, so we'll meet our contract that nil means an error reading the - // file. - if hclFile == nil || hclFile.Body == nil { - return hcl.EmptyBody(), diags - } - - return hclFile.Body, diags -} - -func (p *Parser) parseOverridesFile(file string) hcl.Diagnostics { - - body, diags := p.loadOverrideFile(file) - if body == nil { - return diags - } - - if diags == nil { - diags = hcl.Diagnostics{} - } - - attrs, hclDiags := body.JustAttributes() - diags = safeDiagnosticsExtend(diags, hclDiags) - - for _, attr := range attrs { - - // Grab the expression value. If we have errors performing this we - // cannot continue reliably. - expr, valDiags := attr.Expr.Value(nil) - if valDiags.HasErrors() { - diags = safeDiagnosticsExtend(diags, valDiags) - continue - } - - // Identify whether this variable represents overrides concerned with - // a dependent pack and then handle it accordingly. - isPackVar, packVarDiags := p.isPackVariableObject(attr.Name, expr.Type()) - diags = safeDiagnosticsExtend(diags, packVarDiags) - p.handleOverrideVar(isPackVar, attr, expr) - } - - return diags -} - -func (p *Parser) handleOverrideVar(isPackVar bool, attr *hcl.Attribute, expr cty.Value) { - if isPackVar { - p.handlePackVariableObject(attr.Name, expr, attr.Range) - } else { - v := Variable{ - Name: attr.Name, - Type: expr.Type(), - Value: expr, - DeclRange: attr.Range, - } - p.fileOverrideVars[p.cfg.ParentName] = append(p.fileOverrideVars[p.cfg.ParentName], &v) - } -} - -func (p *Parser) handlePackVariableObject(name string, expr cty.Value, declRange hcl.Range) { - for k := range expr.Type().AttributeTypes() { - av := expr.GetAttr(k) - v := Variable{ - Name: k, - Type: av.Type(), - Value: av, - DeclRange: declRange, - } - p.fileOverrideVars[name] = append(p.fileOverrideVars[name], &v) - } -} - -func (p *Parser) isPackVariableObject(name string, typ cty.Type) (bool, hcl.Diagnostics) { - - // Check whether the name has an associated entry within the root variable - // mapping which indicates whether it's a pack object. - if _, ok := p.cfg.RootVariableFiles[name]; !ok { - return false, nil - } - return typ.IsObjectType(), nil -} diff --git a/internal/pkg/variable/parser/config/config.go b/internal/pkg/variable/parser/config/config.go new file mode 100644 index 00000000..e514cefa --- /dev/null +++ b/internal/pkg/variable/parser/config/config.go @@ -0,0 +1,50 @@ +package config + +import ( + "github.com/hashicorp/nomad-pack/sdk/pack" +) + +type ParserVersion int + +const ( + VUnknown ParserVersion = iota + V1 + V2 +) + +// ParserConfig contains details of the numerous sources of variables which +// should be parsed and merged according to the expected strategy. +type ParserConfig struct { + + // ParserVersion determines which variable parser is loaded to create the + // template context and parse the overrides files. + Version ParserVersion + + // ParentName is the name of the parent pack. Used for deprecated ParserV1. + ParentName string + + // ParentPackID is the PackID of the parent pack. Used for ParserV2 + ParentPackID pack.ID + + // RootVariableFiles contains a map of root variable files, keyed by their + // absolute pack name. "«root pack name».«child pack».«grandchild pack»" + RootVariableFiles map[pack.ID]*pack.File + + // EnvOverrides are key=value variables and take the lowest precedence of + // all sources. If the same key is supplied twice, the last wins. + EnvOverrides map[string]string + + // FileOverrides is a list of files which contain variable overrides in the + // form key=value. The files will be stored before processing to ensure a + // consistent processing experience. Overrides here will replace any + // default root declarations. + FileOverrides []string + + // FlagOverrides are key=value variables and take the highest precedence of + // all sources. If the same key is supplied twice, the last wins. + FlagOverrides map[string]string + + // IgnoreMissingVars determines whether we error or not on variable overrides + // that don't have corresponding vars in the pack. + IgnoreMissingVars bool +} diff --git a/internal/pkg/variable/parser/parsed_variables.go b/internal/pkg/variable/parser/parsed_variables.go new file mode 100644 index 00000000..86ca3bd3 --- /dev/null +++ b/internal/pkg/variable/parser/parsed_variables.go @@ -0,0 +1,211 @@ +package parser + +import ( + "errors" + "slices" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "golang.org/x/exp/maps" +) + +// ParsedVariables wraps the parsed variables returned by parser.Parse and +// provides functionality to access them. +type ParsedVariables struct { + v1Vars map[string]map[string]*variables.Variable + v2Vars map[pack.ID]map[variables.ID]*variables.Variable + Metadata *pack.Metadata + version *config.ParserVersion +} + +func (pv *ParsedVariables) IsV2() bool { + return *pv.version == config.V2 +} + +func (pv *ParsedVariables) IsV1() bool { + return *pv.version == config.V1 +} + +func (pv *ParsedVariables) isLoaded() bool { + return !(pv.version == nil) +} + +// LoadV1Result populates this ParsedVariables with the result from +// parser_v1.Parse(). This function errors if the ParsedVariable has already +// been loaded. +func (pv *ParsedVariables) LoadV1Result(in map[string]map[string]*variables.Variable) error { + if pv.isLoaded() { + return errors.New("already loaded") + } + var vPtr = config.V1 + pv.v1Vars = maps.Clone(in) + pv.version = &vPtr + return nil +} + +// LoadV2Result populates this ParsedVariables with the result from +// parser_v2.Parse(). This function errors if the ParsedVariable has already +// been loaded. +func (pv *ParsedVariables) LoadV2Result(in map[pack.ID]map[variables.ID]*variables.Variable) error { + if pv.isLoaded() { + return errors.New("already loaded") + } + var vPtr = config.V2 + pv.v2Vars = maps.Clone(in) + pv.version = &vPtr + return nil +} + +// GetVars returns the data in the v2 interim data format +func (pv *ParsedVariables) GetVars() map[pack.ID]map[variables.ID]*variables.Variable { + if !pv.isLoaded() { + return nil + } + if *pv.version == config.V1 { + return asV2Vars(pv.v1Vars) + } + return pv.v2Vars +} + +// asV2Vars traverses the v1-style and converts it into an equivalent single +// level v2 variable map +func asV2Vars(in map[string]map[string]*variables.Variable) map[pack.ID]map[variables.ID]*variables.Variable { + var out = make(map[pack.ID]map[variables.ID]*variables.Variable, len(in)) + for k, vs := range in { + out[pack.ID(k)] = make(map[variables.ID]*variables.Variable, len(vs)) + for vk, v := range vs { + out[pack.ID(k)][variables.ID(vk)] = v + } + } + return out +} + +// The following functions generate the appropriate data formats that are sent +// to the text/template renderer for version 1 and version 2 syntax pack +// templates. V1 templates need to have a template context created by the +// `ConvertVariablesToMapInterface` function. V2 templates use a context +// generated by the `ToPackTemplateContext` function + +// SECTION: ParserV2 helper functions + +// ToPackTemplateContext creates a PackTemplateContext from this +// ParsedVariables. +// Even though parsing the variable went without error, it is highly +// possible that conversion to native go types can incur an error. +// If an error is returned, it should be considered terminal. +func (pv *ParsedVariables) ToPackTemplateContext(p *pack.Pack) (PackTemplateContext, hcl.Diagnostics) { + out := make(PackTemplateContext) + diags := pv.toPackTemplateContextR(&out, p) + return out, diags +} + +// toPackTemplateContextR is the recursive implementation of ToPackTemplateContext +func (pv *ParsedVariables) toPackTemplateContextR(tgt *PackTemplateContext, p *pack.Pack) hcl.Diagnostics { + pVars, diags := asMapOfStringToAny(pv.v2Vars[p.VariablesPath()]) + if diags.HasErrors() { + return diags + } + + (*tgt)[CurrentPackKey] = PackData{ + Pack: p, + vars: pVars, + meta: p.Metadata.ConvertToMapInterface(), + } + + for _, d := range p.Dependencies() { + out := make(PackTemplateContext) + diags.Extend(pv.toPackTemplateContextR(&out, d)) + (*tgt)[d.AliasOrName()] = out + } + + return diags +} + +// asMapOfStringToAny builds the map used by the `var` template function +func asMapOfStringToAny(m map[variables.ID]*variables.Variable) (map[string]any, hcl.Diagnostics) { + var diags hcl.Diagnostics + o := make(map[string]any) + for k, cVal := range m { + val, err := variables.ConvertCtyToInterface(cVal.Value) + if err != nil { + diags = packdiags.SafeDiagnosticsAppend(diags, packdiags.DiagFailedToConvertCty(err, cVal.DeclRange.Ptr())) + continue + } + o[string(k)] = val + } + return o, diags +} + +// SECTION: ParserV1 helper functions + +// ConvertVariablesToMapInterface creates the data object for V1 syntax +// templates. +func (pv *ParsedVariables) ConvertVariablesToMapInterface() (map[string]any, hcl.Diagnostics) { + + // Create our output; no matter what we return something. + out := make(map[string]any) + + // Errors can occur when performing the translation. We want to capture all + // of these and return them to the user. This allows them to fix problems + // in a single cycle. + var diags hcl.Diagnostics + + // Iterate each set of pack variable. + for packName, packVars := range pv.v1Vars { + + // packVar collects all variables associated to a pack. + packVar := map[string]any{} + + // Convert each variable and add this to the pack map. + for variableName, variable := range packVars { + varInterface, err := variables.ConvertCtyToInterface(variable.Value) + if err != nil { + diags = packdiags.SafeDiagnosticsAppend(diags, packdiags.DiagFailedToConvertCty(err, variable.DeclRange.Ptr())) + continue + } + packVar[variableName] = varInterface + } + + // Add the pack variable to the full output. + out[packName] = packVar + } + + return out, diags +} + +// SECTION: Generator helper functions + +// AsOverrideFile formats a ParsedVariables so it can be used as a var-file. +// This is used in the `generate varfile` command. +func (pv *ParsedVariables) AsOverrideFile() string { + var out strings.Builder + out.WriteString(pv.varFileHeader()) + + packnames := maps.Keys(pv.v2Vars) + slices.Sort(packnames) + for _, packname := range packnames { + vs := pv.v2Vars[packname] + + varnames := maps.Keys(vs) + slices.Sort(varnames) + for _, varname := range varnames { + v := vs[varname] + out.WriteString(v.AsOverrideString(packname)) + } + } + + return out.String() +} + +// varFileHeader provides additional content to be placed at the top of a +// generated varfile +func (pv *ParsedVariables) varFileHeader() string { + // Use pack metadata to enhance the header if desired. + // _ = vf.Metadata + // This value will be added to the top of the varfile + return "" +} diff --git a/internal/pkg/variable/parser/parser.go b/internal/pkg/variable/parser/parser.go new file mode 100644 index 00000000..a73eb0fc --- /dev/null +++ b/internal/pkg/variable/parser/parser.go @@ -0,0 +1,20 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package parser + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" +) + +type Parser interface { + Parse() (*ParsedVariables, hcl.Diagnostics) +} + +func NewParser(cfg *config.ParserConfig) (Parser, error) { + if cfg.Version == config.V1 { + return NewParserV1(cfg) + } + return NewParserV2(cfg) +} diff --git a/internal/pkg/variable/parser/parser_test.go b/internal/pkg/variable/parser/parser_test.go new file mode 100644 index 00000000..4755c95e --- /dev/null +++ b/internal/pkg/variable/parser/parser_test.go @@ -0,0 +1,110 @@ +package parser + +import ( + "path" + "strings" + "testing" + + "github.com/hashicorp/nomad-pack/internal/pkg/loader" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/shoenig/test/must" +) + +type testPackManagerConfig struct { + Path string + VariableFiles []string + VariableCLIArgs map[string]string + VariableEnvVars map[string]string + UseParserV1 bool +} + +type testPackManager struct { + T *testing.T + cfg *testPackManagerConfig + + // loadedPack is unavailable until the loadAndValidatePacks func is run. + loadedPack *pack.Pack +} + +func newTestPackManager(t *testing.T, path string, useParserV1 bool) *testPackManager { + return &testPackManager{ + T: t, + cfg: &testPackManagerConfig{ + Path: path, + UseParserV1: useParserV1, + }, + } +} + +func (pm *testPackManager) ProcessVariables() *ParsedVariables { + t := pm.T + + // LoadAndValidatePacks uses variables to hide the implementation from publishing + // itself within the test. This has to be all CHONKY because manager depends + // on variables, so we can't create a real pack manager to handle the pack + // state. + var loadAndValidatePacks func() (*pack.Pack, error) = func() (*pack.Pack, error) { + + parentPack, err := loader.Load(pm.cfg.Path) + must.NoError(t, err) + must.NoError(t, parentPack.Validate()) + + // Using the input path to the parent pack, define the path where + // dependencies are stored. + depsPath := path.Join(pm.cfg.Path, "deps") + + // This spectacular line is because the value doesn't exist to be recursed + // until runtime, so we have to declare it here for futire happiness. + var loadAndValidatePackR func(cur *pack.Pack, depsPath string) error + + loadAndValidatePackR = func(cur *pack.Pack, depsPath string) error { + + for _, dep := range cur.Metadata.Dependencies { + // Load and validate the dependency pack. + packPath := path.Join(depsPath, path.Clean(dep.Name)) + depPack, err := loader.Load(packPath) + must.NoError(t, err) + must.NoError(t, depPack.Validate()) + cur.AddDependency(dep.ID(), depPack) + must.NoError(t, loadAndValidatePackR(depPack, path.Join(packPath, "deps"))) + } + + return nil + } + must.NoError(t, loadAndValidatePackR(parentPack, depsPath)) + + return parentPack, nil + } + + loadedPack, err := loadAndValidatePacks() + must.NoError(t, err) + + pm.loadedPack = loadedPack + + // Root vars are nested under the pack name, which is currently the pack name + // without the version. + parentName, _, _ := strings.Cut(path.Base(pm.cfg.Path), "@") + + pCfg := &config.ParserConfig{ + Version: config.V2, + ParentPackID: pack.ID(parentName), + RootVariableFiles: loadedPack.RootVariableFiles(), + EnvOverrides: pm.cfg.VariableEnvVars, + FileOverrides: pm.cfg.VariableFiles, + FlagOverrides: pm.cfg.VariableCLIArgs, + } + + if pm.cfg.UseParserV1 { + pCfg.Version = config.V1 + pCfg.ParentName = parentName + } + + variableParser, err := NewParser(pCfg) + must.NoError(t, err) + + parsedVars, diags := variableParser.Parse() + must.False(t, diags.HasErrors()) + + return parsedVars +} diff --git a/internal/pkg/variable/parser/parser_v1.go b/internal/pkg/variable/parser/parser_v1.go new file mode 100644 index 00000000..63364fe1 --- /dev/null +++ b/internal/pkg/variable/parser/parser_v1.go @@ -0,0 +1,439 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package parser + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/decoder" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/internal/hclhelp" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/schema" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/spf13/afero" + "github.com/zclconf/go-cty/cty" +) + +const VarEnvPrefix = "NOMAD_PACK_VAR_" + +// ParserV1 can parse, merge, and validate HCL variables from multiple different +// sources. +type ParserV1 struct { + fs afero.Afero + cfg *config.ParserConfig + + // rootVars contains all the root variable declared by all parent and child + // packs that are being parsed. The first map is keyed by the pack name, + // the second is by the variable name. + rootVars map[string]map[string]*variables.Variable + + // envOverrideVars, fileOverrideVars, cliOverrideVars are the override + // variables. The maps are keyed by the pack name they are associated to. + envOverrideVars map[string][]*variables.Variable + fileOverrideVars map[string][]*variables.Variable + cliOverrideVars map[string][]*variables.Variable +} + +func NewParserV1(cfg *config.ParserConfig) (*ParserV1, error) { + + // Ensure the parent name is set, otherwise we can't parse correctly. + if cfg.ParentName == "" { + return nil, errors.New("variable parser config requires ParentName to be set") + } + + // Sort the file overrides to ensure variable merging is consistent on + // multiple passes. + sort.Strings(cfg.FileOverrides) + for _, file := range cfg.FileOverrides { + _, err := os.Stat(file) + if err != nil { + return nil, fmt.Errorf("variable file %q not found", file) + } + } + + return &ParserV1{ + fs: afero.Afero{ + Fs: afero.OsFs{}, + }, + cfg: cfg, + rootVars: make(map[string]map[string]*variables.Variable), + envOverrideVars: make(map[string][]*variables.Variable), + fileOverrideVars: make(map[string][]*variables.Variable), + cliOverrideVars: make(map[string][]*variables.Variable), + }, nil +} + +func (p *ParserV1) Parse() (*ParsedVariables, hcl.Diagnostics) { + + // Parse the root variables. If we encounter an error here, we are unable + // to reliably continue. + diags := p.parseRootFiles() + if diags.HasErrors() { + return nil, diags + } + + // Parse env, file, and CLI overrides. + for k, v := range p.cfg.EnvOverrides { + envOverrideDiags := p.parseEnvVariable(k, v) + diags = packdiags.SafeDiagnosticsExtend(diags, envOverrideDiags) + } + + for _, fileOverride := range p.cfg.FileOverrides { + fileOverrideDiags := p.parseOverridesFile(fileOverride) + diags = packdiags.SafeDiagnosticsExtend(diags, fileOverrideDiags) + } + + for k, v := range p.cfg.FlagOverrides { + cliOverrideDiags := p.parseCLIVariable(k, v) + diags = packdiags.SafeDiagnosticsExtend(diags, cliOverrideDiags) + } + + if diags.HasErrors() { + return nil, diags + } + + // Iterate all our override variables and merge these into our root + // variables with the CLI taking highest priority. + for _, override := range []map[string][]*variables.Variable{p.envOverrideVars, p.fileOverrideVars, p.cliOverrideVars} { + for packName, variables := range override { + for _, v := range variables { + existing, exists := p.rootVars[packName][v.Name.String()] + if !exists { + if !p.cfg.IgnoreMissingVars { + diags = diags.Append(packdiags.DiagMissingRootVar(v.Name.String(), v.DeclRange.Ptr())) + } + continue + } + if mergeDiags := existing.Merge(v); mergeDiags.HasErrors() { + diags = diags.Extend(mergeDiags) + } + } + } + } + out := new(ParsedVariables) + out.LoadV1Result(p.rootVars) + return out, diags +} + +func (p *ParserV1) loadOverrideFile(file string) (hcl.Body, hcl.Diagnostics) { + + src, err := p.fs.ReadFile(file) + // FIXME - Workaround for ending heredoc with no linefeed. + // Variables files shouldn't care about the extra linefeed, but jamming one + // in all the time feels bad. + src = append(src, "\n"...) + if err != nil { + return nil, hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Failed to read file", + Detail: fmt.Sprintf("The file %q could not be read.", file), + }, + } + } + + return p.loadPackFile(&pack.File{Path: file, Content: src}) +} + +// loadPackFile takes a pack.File and parses this using a hclparse.Parser. The +// file can be either HCL and JSON format. +func (p *ParserV1) loadPackFile(file *pack.File) (hcl.Body, hcl.Diagnostics) { + + var ( + hclFile *hcl.File + diags hcl.Diagnostics + ) + + // Instantiate a new parser each time. Using the same parser where variable + // names collide from different packs will cause problems. + hclParser := hclparse.NewParser() + + // Depending on the fix extension, use the correct HCL parser. + switch { + case strings.HasSuffix(file.Name, ".json"): + hclFile, diags = hclParser.ParseJSON(file.Content, file.Path) + default: + hclFile, diags = hclParser.ParseHCL(file.Content, file.Path) + } + + // If the returned file or body is nil, then we'll return a non-nil empty + // body, so we'll meet our contract that nil means an error reading the + // file. + if hclFile == nil || hclFile.Body == nil { + return hcl.EmptyBody(), diags + } + + return hclFile.Body, diags +} + +func (p *ParserV1) parseOverridesFile(file string) hcl.Diagnostics { + + body, diags := p.loadOverrideFile(file) + if body == nil { + return diags + } + + if diags == nil { + diags = hcl.Diagnostics{} + } + + attrs, hclDiags := body.JustAttributes() + diags = packdiags.SafeDiagnosticsExtend(diags, hclDiags) + + for _, attr := range attrs { + + // Grab the expression value. If we have errors performing this we + // cannot continue reliably. + expr, valDiags := attr.Expr.Value(nil) + if valDiags.HasErrors() { + diags = packdiags.SafeDiagnosticsExtend(diags, valDiags) + continue + } + + // Identify whether this variable represents overrides concerned with + // a dependent pack and then handle it accordingly. + isPackVar, packVarDiags := p.isPackVariableObject(pack.ID(attr.Name), expr.Type()) + diags = packdiags.SafeDiagnosticsExtend(diags, packVarDiags) + p.handleOverrideVar(isPackVar, attr, expr) + } + + return diags +} + +func (p *ParserV1) handleOverrideVar(isPackVar bool, attr *hcl.Attribute, expr cty.Value) { + if isPackVar { + p.handlePackVariableObject(attr.Name, expr, attr.Range) + } else { + v := variables.Variable{ + Name: variables.ID(attr.Name), + Type: expr.Type(), + Value: expr, + DeclRange: attr.Range, + } + p.fileOverrideVars[p.cfg.ParentName] = append(p.fileOverrideVars[p.cfg.ParentName], &v) + } +} + +func (p *ParserV1) handlePackVariableObject(name string, expr cty.Value, declRange hcl.Range) { + for k := range expr.Type().AttributeTypes() { + av := expr.GetAttr(k) + v := variables.Variable{ + Name: variables.ID(k), + Type: av.Type(), + Value: av, + DeclRange: declRange, + } + p.fileOverrideVars[name] = append(p.fileOverrideVars[name], &v) + } +} + +func (p *ParserV1) isPackVariableObject(name pack.ID, typ cty.Type) (bool, hcl.Diagnostics) { + + // Check whether the name has an associated entry within the root variable + // mapping which indicates whether it's a pack object. + if _, ok := p.cfg.RootVariableFiles[name]; !ok { + return false, nil + } + return typ.IsObjectType(), nil +} + +func (p *ParserV1) parseEnvVariable(name string, rawVal string) hcl.Diagnostics { + // Split the name to see if we have a namespace CLI variable for a child + // pack and set the default packVarName. + splitName := strings.SplitN(name, ".", 2) + packVarName := []string{p.cfg.ParentName, name} + + switch len(splitName) { + case 1: + // Fallthrough, nothing to do or see. + case 2: + // We are dealing with a namespaced variable. Overwrite the preset + // values of packVarName. + packVarName[0] = splitName[0] + packVarName[1] = splitName[1] + default: + // We cannot handle a splitName where the variable includes more than + // one separator. + return hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Invalid %s option", strings.TrimRight(VarEnvPrefix, "_")), + Detail: fmt.Sprintf("The given environment variable %s%s=%s is not correctly specified. The variable name must not have more than one dot `.` separator.", VarEnvPrefix, name, rawVal), + }, + } + } + + // Generate a filename based on the CLI var, so we have some context for any + // HCL diagnostics. + fakeRange := hcl.Range{Filename: fmt.Sprintf("", name)} + + // If the variable has not been configured in the root then ignore it. This + // is a departure from the way in which flags and var-files are handled. + // The environment might contain NOMAD_PACK_VAR variables used for other + // packs that might be run on the same system but are not used with this + // particular pack. + existing, exists := p.rootVars[packVarName[0]][packVarName[1]] + if !exists { + return nil + } + + expr, diags := hclhelp.ExpressionFromVariableDefinition(fakeRange.Filename, rawVal, existing.Type) + if diags.HasErrors() { + return diags + } + + val, diags := expr.Value(nil) + if diags.HasErrors() { + return diags + } + + // If our stored type isn't cty.NilType then attempt to covert the override + // variable, so we know they are compatible. + if existing.Type != cty.NilType { + var err *hcl.Diagnostic + val, err = hclhelp.ConvertValUsingType(val, existing.Type, expr.Range().Ptr()) + if err != nil { + return hcl.Diagnostics{err} + } + } + + // We have a verified override variable. + v := variables.Variable{ + Name: variables.ID(packVarName[1]), + Type: val.Type(), + Value: val, + DeclRange: fakeRange, + } + p.envOverrideVars[packVarName[0]] = append(p.envOverrideVars[packVarName[0]], &v) + + return nil +} + +func (p *ParserV1) parseCLIVariable(name string, rawVal string) hcl.Diagnostics { + // Split the name to see if we have a namespace CLI variable for a child + // pack and set the default packVarName. + splitName := strings.SplitN(name, ".", 2) + packVarName := []string{p.cfg.ParentName, name} + + switch len(splitName) { + case 1: + // Fallthrough, nothing to do or see. + case 2: + // We are dealing with a namespaced variable. Overwrite the preset + // values of packVarName. + packVarName[0] = splitName[0] + packVarName[1] = splitName[1] + default: + // We cannot handle a splitName where the variable includes more than + // one separator. + return hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: "Invalid -var option", + Detail: fmt.Sprintf("The given -var option %s=%s is not correctly specified. The variable name must not have more than one dot `.` separator.", name, rawVal), + }, + } + } + + // Generate a filename based on the CLI var, so we have some context for any + // HCL diagnostics. + fakeRange := hcl.Range{Filename: fmt.Sprintf("", name)} + + // If the variable has not been configured in the root then exit. This is a + // standard requirement, especially because we would be unable to ensure a + // consistent type. + existing, exists := p.rootVars[packVarName[0]][packVarName[1]] + if !exists { + return hcl.Diagnostics{packdiags.DiagMissingRootVar(name, &fakeRange)} + } + + expr, diags := hclhelp.ExpressionFromVariableDefinition(fakeRange.Filename, rawVal, existing.Type) + if diags.HasErrors() { + return diags + } + + val, diags := expr.Value(nil) + if diags.HasErrors() { + return diags + } + + // If our stored type isn't cty.NilType then attempt to covert the override + // variable, so we know they are compatible. + if existing.Type != cty.NilType { + var err *hcl.Diagnostic + val, err = hclhelp.ConvertValUsingType(val, existing.Type, expr.Range().Ptr()) + if err != nil { + return hcl.Diagnostics{err} + } + } + + // We have a verified override variable. + v := variables.Variable{ + Name: variables.ID(packVarName[1]), + Type: val.Type(), + Value: val, + DeclRange: fakeRange, + } + p.cliOverrideVars[packVarName[0]] = append(p.cliOverrideVars[packVarName[0]], &v) + + return nil +} + +func (p *ParserV1) parseRootFiles() hcl.Diagnostics { + + var diags hcl.Diagnostics + + // Iterate all our root variable files. + for name, file := range p.cfg.RootVariableFiles { + + hclBody, loadDiags := p.loadPackFile(file) + diags = packdiags.SafeDiagnosticsExtend(diags, loadDiags) + + content, contentDiags := hclBody.Content(schema.VariableFileSchema) + diags = packdiags.SafeDiagnosticsExtend(diags, contentDiags) + + rootVars, parseDiags := p.parseRootBodyContent(content) + diags = packdiags.SafeDiagnosticsExtend(diags, parseDiags) + + // If we don't have any errors processing the file, and it's content, + // add an entry. + if !diags.HasErrors() { + // The v2 loader returns pack names in dotted ancestor form, + // grap the last element of the string + parts := strings.Split(name.String(), ".") + name := parts[len(parts)-1] + p.rootVars[name] = rootVars + } + } + + return diags +} + +// parseRootBodyContent process the body of a root variables file, parsing +// each variable block found. +func (p *ParserV1) parseRootBodyContent(body *hcl.BodyContent) (map[string]*variables.Variable, hcl.Diagnostics) { + + packRootVars := map[string]*variables.Variable{} + + var diags hcl.Diagnostics + + // Due to the parsing that uses variableFileSchema, it is safe to assume + // every block has a type "variable". + for _, block := range body.Blocks { + cfg, cfgDiags := decoder.DecodeVariableBlock(block) + diags = packdiags.SafeDiagnosticsExtend(diags, cfgDiags) + if cfg != nil { + packRootVars[cfg.Name.String()] = cfg + } + } + return packRootVars, diags +} diff --git a/internal/pkg/variable/cli_test.go b/internal/pkg/variable/parser/parser_v1_test.go similarity index 50% rename from internal/pkg/variable/cli_test.go rename to internal/pkg/variable/parser/parser_v1_test.go index 54a7ae36..e93c1438 100644 --- a/internal/pkg/variable/cli_test.go +++ b/internal/pkg/variable/parser/parser_v1_test.go @@ -1,37 +1,38 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -package variable +package parser import ( "fmt" - "os" - "path" "testing" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" "github.com/shoenig/test/must" "github.com/spf13/afero" "github.com/zclconf/go-cty/cty" ) -func TestParser_parseCLIVariable(t *testing.T) { +func TestParserV1_parseCLIVariable(t *testing.T) { testCases := []struct { - inputParser *Parser + inputParser *ParserV1 inputName string inputRawVal string expectedError bool - expectedCLIVars map[string][]*Variable - expectedEnvVars map[string][]*Variable + expectedCLIVars map[string][]*variables.Variable + expectedEnvVars map[string][]*variables.Variable name string }{ { - inputParser: &Parser{ + inputParser: &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{ + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{ "example": { - "region": &Variable{ + "region": &variables.Variable{ Name: "region", Type: cty.String, Value: cty.StringVal("vlc"), @@ -39,13 +40,13 @@ func TestParser_parseCLIVariable(t *testing.T) { }, }, }, - cliOverrideVars: make(map[string][]*Variable), - envOverrideVars: make(map[string][]*Variable), + cliOverrideVars: make(map[string][]*variables.Variable), + envOverrideVars: make(map[string][]*variables.Variable), }, inputName: "region", inputRawVal: "vlc", expectedError: false, - expectedCLIVars: map[string][]*Variable{ + expectedCLIVars: map[string][]*variables.Variable{ "example": { { Name: "region", @@ -55,16 +56,16 @@ func TestParser_parseCLIVariable(t *testing.T) { }, }, }, - expectedEnvVars: make(map[string][]*Variable), + expectedEnvVars: make(map[string][]*variables.Variable), name: "non-namespaced variable", }, { - inputParser: &Parser{ + inputParser: &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{ + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{ "example": { - "region": &Variable{ + "region": &variables.Variable{ Name: "region", Type: cty.String, Value: cty.StringVal("vlc"), @@ -72,12 +73,12 @@ func TestParser_parseCLIVariable(t *testing.T) { }, }, }, - cliOverrideVars: make(map[string][]*Variable), + cliOverrideVars: make(map[string][]*variables.Variable), }, inputName: "example.region", inputRawVal: "vlc", expectedError: false, - expectedCLIVars: map[string][]*Variable{ + expectedCLIVars: map[string][]*variables.Variable{ "example": { { Name: "region", @@ -90,25 +91,25 @@ func TestParser_parseCLIVariable(t *testing.T) { name: "namespaced variable", }, { - inputParser: &Parser{ + inputParser: &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{}, - cliOverrideVars: make(map[string][]*Variable), + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{}, + cliOverrideVars: make(map[string][]*variables.Variable), }, inputName: "example.region", inputRawVal: "vlc", expectedError: true, - expectedCLIVars: map[string][]*Variable{}, + expectedCLIVars: map[string][]*variables.Variable{}, name: "root variable absent", }, { - inputParser: &Parser{ + inputParser: &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{ + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{ "example": { - "region": &Variable{ + "region": &variables.Variable{ Name: "region", Type: cty.DynamicPseudoType, Value: cty.MapVal(map[string]cty.Value{ @@ -118,12 +119,12 @@ func TestParser_parseCLIVariable(t *testing.T) { }, }, }, - cliOverrideVars: make(map[string][]*Variable), + cliOverrideVars: make(map[string][]*variables.Variable), }, inputName: "example.region", inputRawVal: "vlc", expectedError: true, - expectedCLIVars: map[string][]*Variable{}, + expectedCLIVars: map[string][]*variables.Variable{}, name: "unconvertable variable", }, } @@ -139,75 +140,75 @@ func TestParser_parseCLIVariable(t *testing.T) { } } -func TestParser_parseHeredocAtEOF(t *testing.T) { - inputParser := &Parser{ +func TestParserV1_parseHeredocAtEOF(t *testing.T) { + inputParser := &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{}, - cliOverrideVars: make(map[string][]*Variable), + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{}, + cliOverrideVars: make(map[string][]*variables.Variable), } - fixturePath := Fixture("variable_test/heredoc.vars.hcl") + fixturePath := testfixture.AbsPath(t, "v1/variable_test/heredoc.vars.hcl") b, diags := inputParser.loadOverrideFile(fixturePath) must.NotNil(t, b) must.SliceEmpty(t, diags) } -func TestParser_VariableOverrides(t *testing.T) { +func TestParserV1_VariableOverrides(t *testing.T) { testcases := []struct { Name string - Parser *Parser + Parser *ParserV1 Expect string }{ { Name: "no override", - Parser: NewTestInputParser(), + Parser: NewTestInputParserV1(), Expect: "root", }, { Name: "env override", - Parser: NewTestInputParser(WithEnvVar("input", "env")), + Parser: NewTestInputParserV1(WithEnvVarV1("input", "env")), Expect: "env", }, { Name: "file override", - Parser: NewTestInputParser(WithFileVar("input", "file")), + Parser: NewTestInputParserV1(WithFileVarV1("input", "file")), Expect: "file", }, { Name: "flag override", - Parser: NewTestInputParser(WithCliVar("input", "flag")), + Parser: NewTestInputParserV1(WithCliVarV1("input", "flag")), Expect: "flag", }, { Name: "file opaques env", - Parser: NewTestInputParser( - WithEnvVar("input", "env"), - WithFileVar("input", "file"), + Parser: NewTestInputParserV1( + WithEnvVarV1("input", "env"), + WithFileVarV1("input", "file"), ), Expect: "file", }, { Name: "flag opaques env", - Parser: NewTestInputParser( - WithEnvVar("input", "env"), - WithCliVar("input", "flag"), + Parser: NewTestInputParserV1( + WithEnvVarV1("input", "env"), + WithCliVarV1("input", "flag"), ), Expect: "flag", }, { Name: "flag opaques file", - Parser: NewTestInputParser( - WithFileVar("input", "file"), - WithCliVar("input", "flag"), + Parser: NewTestInputParserV1( + WithFileVarV1("input", "file"), + WithCliVarV1("input", "flag"), ), Expect: "flag", }, { Name: "flag opaques env and file", - Parser: NewTestInputParser( - WithEnvVar("input", "env"), - WithFileVar("input", "file"), - WithCliVar("input", "flag"), + Parser: NewTestInputParserV1( + WithEnvVarV1("input", "env"), + WithFileVarV1("input", "file"), + WithCliVarV1("input", "flag"), ), Expect: "flag", }, @@ -219,45 +220,48 @@ func TestParser_VariableOverrides(t *testing.T) { must.NotNil(t, pv) must.SliceEmpty(t, diags) - must.Eq(t, tc.Expect, pv.Vars["example"]["input"].Value.AsString()) + must.Eq(t, tc.Expect, pv.v1Vars["example"]["input"].Value.AsString()) }) } } -func Fixture(fPath string) string { - // FIXME: Find the fixture folder in a less janky way - cwd, _ := os.Getwd() - return path.Join(cwd, "../../../fixtures/", fPath) +func TestParserV1_parseNestedPack(t *testing.T) { + fixturePath := testfixture.AbsPath(t, "v1/test_registry/packs/my_alias_test") + pm := newTestPackManager(t, fixturePath, true) + + pvs := pm.ProcessVariables() + must.NotNil(t, pvs) + must.MapNotEmpty(t, pvs.v1Vars) } -type TestParserOption func(*Parser) +type testParserV1Option func(*ParserV1) -func WithEnvVar(key, value string) TestParserOption { - return func(p *Parser) { - p.envOverrideVars["example"] = append(p.envOverrideVars["example"], NewStringVariable(key, value, "env")) +func WithEnvVarV1(key, value string) testParserV1Option { + return func(p *ParserV1) { + p.envOverrideVars["example"] = append(p.envOverrideVars["example"], NewStringVariableV1(key, value, "env")) } } -func WithCliVar(key, value string) TestParserOption { - return func(p *Parser) { - p.cliOverrideVars["example"] = append(p.cliOverrideVars["example"], NewStringVariable(key, value, "cli")) +func WithCliVarV1(key, value string) testParserV1Option { + return func(p *ParserV1) { + p.cliOverrideVars["example"] = append(p.cliOverrideVars["example"], NewStringVariableV1(key, value, "cli")) } } -func WithFileVar(key, value string) TestParserOption { - return func(p *Parser) { - p.cliOverrideVars["example"] = append(p.cliOverrideVars["example"], NewStringVariable(key, value, "file")) +func WithFileVarV1(key, value string) testParserV1Option { + return func(p *ParserV1) { + p.cliOverrideVars["example"] = append(p.cliOverrideVars["example"], NewStringVariableV1(key, value, "file")) } } -func NewTestInputParser(opts ...TestParserOption) *Parser { +func NewTestInputParserV1(opts ...testParserV1Option) *ParserV1 { - p := &Parser{ + p := &ParserV1{ fs: afero.Afero{Fs: afero.OsFs{}}, - cfg: &ParserConfig{ParentName: "example"}, - rootVars: map[string]map[string]*Variable{ + cfg: &config.ParserConfig{ParentName: "example"}, + rootVars: map[string]map[string]*variables.Variable{ "example": { - "input": &Variable{ + "input": &variables.Variable{ Name: "input", Type: cty.String, Value: cty.StringVal("root"), @@ -265,9 +269,9 @@ func NewTestInputParser(opts ...TestParserOption) *Parser { }, }, }, - envOverrideVars: make(map[string][]*Variable), - fileOverrideVars: make(map[string][]*Variable), - cliOverrideVars: make(map[string][]*Variable), + envOverrideVars: make(map[string][]*variables.Variable), + fileOverrideVars: make(map[string][]*variables.Variable), + cliOverrideVars: make(map[string][]*variables.Variable), } // Loop through each option @@ -278,9 +282,9 @@ func NewTestInputParser(opts ...TestParserOption) *Parser { return p } -func NewStringVariable(key, value, kind string) *Variable { - return &Variable{ - Name: key, +func NewStringVariableV1(key, value, kind string) *variables.Variable { + return &variables.Variable{ + Name: variables.ID(key), Type: cty.String, Value: cty.StringVal(value), DeclRange: hcl.Range{Filename: fmt.Sprintf("", key, kind)}, diff --git a/internal/pkg/variable/parser/parser_v2.go b/internal/pkg/variable/parser/parser_v2.go new file mode 100644 index 00000000..9b77f134 --- /dev/null +++ b/internal/pkg/variable/parser/parser_v2.go @@ -0,0 +1,326 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package parser + +import ( + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/hashicorp/nomad-pack/internal/pkg/varfile" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/decoder" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/envloader" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/internal/hclhelp" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/schema" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/spf13/afero" + "github.com/zclconf/go-cty/cty" +) + +type ParserV2 struct { + fs afero.Afero + cfg *config.ParserConfig + + // rootVars contains all the root variable declared by all parent and child + // packs that are being parsed. The first map is keyed by the pack name, + // the second is by the variable name. + rootVars map[pack.ID]map[variables.ID]*variables.Variable + + // envOverrideVars, fileOverrideVars, cliOverrideVars are the override + // variables. The maps are keyed by the pack name they are associated to. + envOverrideVars variables.PackIDKeyedVarMap + fileOverrideVars variables.PackIDKeyedVarMap + flagOverrideVars variables.PackIDKeyedVarMap +} + +func NewParserV2(cfg *config.ParserConfig) (*ParserV2, error) { + + // Ensure the parent name is set, otherwise we can't parse correctly. + if cfg.ParentPackID == "" { + return nil, errors.New("variable parser config requires ParentName to be set") + } + + // Sort the file overrides to ensure variable merging is consistent on + // multiple passes. + sort.Strings(cfg.FileOverrides) + for _, file := range cfg.FileOverrides { + _, err := os.Stat(file) + if err != nil { + return nil, fmt.Errorf("variable file %q not found", file) + } + } + + return &ParserV2{ + fs: afero.Afero{ + Fs: afero.OsFs{}, + }, + cfg: cfg, + rootVars: make(map[pack.ID]map[variables.ID]*variables.Variable), + envOverrideVars: make(variables.PackIDKeyedVarMap), + fileOverrideVars: make(variables.PackIDKeyedVarMap), + flagOverrideVars: make(variables.PackIDKeyedVarMap), + }, nil +} + +func (p *ParserV2) Parse() (*ParsedVariables, hcl.Diagnostics) { + + // Parse the root variables. If we encounter an error here, we are unable + // to reliably continue. + diags := p.parseRootFiles() + if diags.HasErrors() { + return nil, diags + } + + // Parse env, file, and CLI overrides. + for k, v := range p.cfg.EnvOverrides { + envOverrideDiags := p.parseEnvVariable(k, v) + diags = packdiags.SafeDiagnosticsExtend(diags, envOverrideDiags) + } + + for _, fileOverride := range p.cfg.FileOverrides { + _, fileOverrideDiags := p.newParseOverridesFile(fileOverride) + diags = packdiags.SafeDiagnosticsExtend(diags, fileOverrideDiags) + } + + for k, v := range p.cfg.FlagOverrides { + flagOverrideDiags := p.parseFlagVariable(k, v) + diags = packdiags.SafeDiagnosticsExtend(diags, flagOverrideDiags) + } + + if diags.HasErrors() { + return nil, diags + } + + // Iterate all our override variables and merge these into our root + // variables with the CLI taking highest priority. + for _, override := range []variables.PackIDKeyedVarMap{p.envOverrideVars, p.fileOverrideVars, p.flagOverrideVars} { + for packName, variables := range override { + for _, v := range variables { + existing, exists := p.rootVars[packName][v.Name] + if !exists { + if !p.cfg.IgnoreMissingVars { + diags = diags.Append(packdiags.DiagMissingRootVar(v.Name.String(), v.DeclRange.Ptr())) + } + continue + } + if mergeDiags := existing.Merge(v); mergeDiags.HasErrors() { + diags = diags.Extend(mergeDiags) + } + } + } + } + + out := new(ParsedVariables) + out.LoadV2Result(p.rootVars) + return out, diags +} + +func (p *ParserV2) newParseOverridesFile(file string) (map[string]*hcl.File, hcl.Diagnostics) { + var diags hcl.Diagnostics + + src, err := p.fs.ReadFile(file) + if err != nil { + return nil, diags.Append(packdiags.DiagFileNotFound(file)) + } + + ovrds := make(variables.Overrides) + + // Decode into the local recipient object + if hfm, vfDiags := varfile.Decode(file, src, nil, &ovrds); vfDiags.HasErrors() { + return hfm, vfDiags.Extend(diags) + } + for _, o := range ovrds[pack.ID(file)] { + // Identify whether this variable override is for a dependency pack + // and then handle it accordingly. + p.newHandleOverride(o) + } + return nil, diags +} + +func (p *ParserV2) newHandleOverride(o *variables.Override) { + // Is Pack Variable Object? + // Check whether the name has an associated entry within the root variable + // mapping which indicates whether it's a pack object. + if _, ok := p.cfg.RootVariableFiles[o.Path]; ok { + p.newHandleOverrideVar(o) + } +} + +func (p *ParserV2) newHandleOverrideVar(o *variables.Override) { + v := variables.Variable{ + Name: o.Name, + Type: o.Type, + Value: o.Value, + DeclRange: o.Range, + } + p.fileOverrideVars[o.Path] = append(p.fileOverrideVars[o.Path], &v) +} + +// loadPackFile takes a pack.File and parses this using a hclparse.Parser. The +// file can be either HCL and JSON format. +func (p *ParserV2) loadPackFile(file *pack.File) (hcl.Body, hcl.Diagnostics) { + + var ( + hclFile *hcl.File + diags hcl.Diagnostics + ) + + // Instantiate a new parser each time. Using the same parser where variable + // names collide from different packs will cause problems. + hclParser := hclparse.NewParser() + + // Depending on the fix extension, use the correct HCL parser. + switch { + case strings.HasSuffix(file.Name, ".json"): + hclFile, diags = hclParser.ParseJSON(file.Content, file.Path) + default: + hclFile, diags = hclParser.ParseHCL(file.Content, file.Path) + } + + // If the returned file or body is nil, then we'll return a non-nil empty + // body, so we'll meet our contract that nil means an error reading the + // file. + if hclFile == nil || hclFile.Body == nil { + return hcl.EmptyBody(), diags + } + + return hclFile.Body, diags +} + +func (p *ParserV2) parseRootFiles() hcl.Diagnostics { + + var diags hcl.Diagnostics + + // Iterate all our root variable files. + for name, file := range p.cfg.RootVariableFiles { + + hclBody, loadDiags := p.loadPackFile(file) + diags = packdiags.SafeDiagnosticsExtend(diags, loadDiags) + + content, contentDiags := hclBody.Content(schema.VariableFileSchema) + diags = packdiags.SafeDiagnosticsExtend(diags, contentDiags) + + rootVars, parseDiags := p.parseRootBodyContent(content) + diags = packdiags.SafeDiagnosticsExtend(diags, parseDiags) + + // If we don't have any errors processing the file, and its content, + // add an entry. + if !diags.HasErrors() { + p.rootVars[name] = rootVars + } + } + + return diags +} + +// parseRootBodyContent process the body of a root variables file, parsing +// each variable block found. +func (p *ParserV2) parseRootBodyContent(body *hcl.BodyContent) (map[variables.ID]*variables.Variable, hcl.Diagnostics) { + + packRootVars := map[variables.ID]*variables.Variable{} + + var diags hcl.Diagnostics + + // Due to the parsing that uses variableFileSchema, it is safe to assume + // every block has a type "variable". + for _, block := range body.Blocks { + cfg, cfgDiags := decoder.DecodeVariableBlock(block) + diags = packdiags.SafeDiagnosticsExtend(diags, cfgDiags) + if cfg != nil { + packRootVars[cfg.Name] = cfg + } + } + return packRootVars, diags +} + +func (p *ParserV2) parseEnvVariable(name string, rawVal string) hcl.Diagnostics { + return p.parseVariableImpl(name, rawVal, p.envOverrideVars, name, "environment") + +} +func (p *ParserV2) parseFlagVariable(name string, rawVal string) hcl.Diagnostics { + return p.parseVariableImpl(name, rawVal, p.flagOverrideVars, "-var", "arguments") +} + +func (p *ParserV2) parseVariableImpl(name, rawVal string, tgt variables.PackIDKeyedVarMap, typeTxt, rangeDesc string) hcl.Diagnostics { + if rangeDesc == "environment" { + name = strings.TrimPrefix(name, envloader.DefaultPrefix) + } + + // Split the name to see if we have a namespace CLI variable for a child + // pack and set the default packVarName. + splitName := strings.Split(name, ".") + + if len(splitName) < 2 || splitName[0] != p.cfg.ParentPackID.String() { + return hcl.Diagnostics{ + { + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Invalid %s option %s=%s", typeTxt, name, rawVal), + Detail: fmt.Sprintf("The given %s option %s=%s is not correctly specified.\nVariable names must be dot-separated, absolute paths to a variable including the root pack name %q.", typeTxt, name, rawVal, p.cfg.ParentPackID), + }, + } + } + + // Generate a filename based on the incoming var, so we have some context for + // any HCL diagnostics. + + // Get a reasonable count for the lines in the provided value. You'd think + // these had to be flat, but naaah. + lines := strings.Split(rawVal, "\n") + lc := len(lines) + endCol := len(lines[lc-1]) + + fakeRange := hcl.Range{ + Filename: fmt.Sprintf("", name, rangeDesc), + Start: hcl.Pos{Line: 1, Column: 1, Byte: 0}, + End: hcl.Pos{Line: lc, Column: endCol, Byte: len(rawVal)}, + } + + varPID := pack.ID(strings.Join(splitName[:len(splitName)-1], ".")) + varVID := variables.ID(splitName[len(splitName)-1]) + // If the variable has not been configured in the root then exit. This is a + // standard requirement, especially because we would be unable to ensure a + // consistent type. + existing, exists := p.rootVars[varPID][varVID] + if !exists { + return hcl.Diagnostics{packdiags.DiagMissingRootVar(name, &fakeRange)} + } + + expr, diags := hclhelp.ExpressionFromVariableDefinition(fakeRange.Filename, rawVal, existing.Type) + if diags.HasErrors() { + return diags + } + + val, diags := expr.Value(nil) + if diags.HasErrors() { + return diags + } + + // If our stored type isn't cty.NilType then attempt to covert the override + // variable, so we know they are compatible. + if existing.Type != cty.NilType { + var err *hcl.Diagnostic + val, err = hclhelp.ConvertValUsingType(val, existing.Type, expr.Range().Ptr()) + if err != nil { + return hcl.Diagnostics{err} + } + } + + // We have a verified override variable. + v := variables.Variable{ + Name: varVID, + Type: val.Type(), + Value: val, + DeclRange: fakeRange, + } + tgt[varPID] = append(tgt[varPID], &v) + + return nil +} diff --git a/internal/pkg/variable/parser/parser_v2_test.go b/internal/pkg/variable/parser/parser_v2_test.go new file mode 100644 index 00000000..fa32fbcf --- /dev/null +++ b/internal/pkg/variable/parser/parser_v2_test.go @@ -0,0 +1,450 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package parser + +import ( + "fmt" + "path" + "strings" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/internal/pkg/loader" + "github.com/hashicorp/nomad-pack/internal/pkg/testfixture" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/envloader" + "github.com/hashicorp/nomad-pack/internal/pkg/variable/parser/config" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/hashicorp/nomad-pack/sdk/pack/variables" + "github.com/shoenig/test/must" + "github.com/spf13/afero" + "github.com/zclconf/go-cty/cty" +) + +func TestParserV2_parseFlagVariable(t *testing.T) { + testCases := []struct { + inputParser *ParserV2 + inputName string + inputRawVal string + expectedError bool + expectedFlagVars variables.PackIDKeyedVarMap + expectedEnvVars variables.PackIDKeyedVarMap + name string + }{ + { + name: "non-namespaced variable", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + }, + inputName: "region", + inputRawVal: "vlc", + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + expectedEnvVars: make(variables.PackIDKeyedVarMap), + }, + { + name: "namespaced variable", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + }, + inputName: "example.region", + inputRawVal: "vlc", + expectedError: false, + expectedFlagVars: variables.PackIDKeyedVarMap{ + "example": { + { + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + }, + { + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{}, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + }, + inputName: "example.region", + inputRawVal: "vlc", + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + name: "root variable absent", + }, + { + name: "unconvertable variable", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.DynamicPseudoType, + Value: cty.MapVal(map[string]cty.Value{ + "region": cty.StringVal("dc1"), + }), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + }, + inputName: "example.region", + inputRawVal: "vlc", + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc := tc + actualErr := tc.inputParser.parseFlagVariable(tc.inputName, tc.inputRawVal) + + if tc.expectedError { + must.NotNil(t, actualErr) + return + } + must.Nil(t, actualErr, must.Sprintf("actualErr: %v", actualErr)) + must.MapEq(t, tc.expectedFlagVars, tc.inputParser.flagOverrideVars) + }) + } +} + +func TestParserV2_parseEnvVariable(t *testing.T) { + type testCase struct { + inputParser *ParserV2 + envKey string + envValue string + expectedError bool + expectedFlagVars variables.PackIDKeyedVarMap + expectedEnvVars variables.PackIDKeyedVarMap + name string + } + + withDefault := func(e, d string) string { + t.Helper() + if e == "" { + return d + } + return e + } + + getEnvKey := func(tc testCase) string { + t.Helper() + return withDefault(tc.envKey, "NOMAD_PACK_VAR_example.region") + } + + getEnvValue := func(tc testCase) string { + t.Helper() + return withDefault(tc.envValue, "vlc") + } + + setTestEnvKeyForVar := func(t *testing.T, tc testCase) string { + t.Helper() + var k string = getEnvKey(tc) + var v string = getEnvValue(tc) + t.Logf("setting %s to %s", k, v) + t.Setenv(k, v) + return strings.TrimPrefix(k, envloader.DefaultPrefix) + } + + testCases := []testCase{ + { + name: "non-namespaced variable", + envKey: "NOMAD_PACK_VAR_region", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + }, + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + expectedEnvVars: make(variables.PackIDKeyedVarMap), + }, + { + name: "namespaced variable", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + }, + expectedError: false, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + expectedEnvVars: variables.PackIDKeyedVarMap{ + "example": { + { + Name: "region", + Type: cty.String, + Value: cty.StringVal("vlc"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + }, + { + name: "root variable absent", + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{}, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + }, + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + expectedEnvVars: variables.PackIDKeyedVarMap{}, + }, + { + name: "unconvertable variable", + envKey: "NOMAD_PACK_VAR_example.region", + envValue: `{region: "dc1}`, + inputParser: &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "region": &variables.Variable{ + Name: "region", + Type: cty.DynamicPseudoType, + Value: cty.MapVal(map[string]cty.Value{ + "region": cty.StringVal("dc1"), + }), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + flagOverrideVars: make(variables.PackIDKeyedVarMap), + envOverrideVars: make(variables.PackIDKeyedVarMap), + }, + expectedError: true, + expectedFlagVars: variables.PackIDKeyedVarMap{}, + expectedEnvVars: variables.PackIDKeyedVarMap{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mapKey := setTestEnvKeyForVar(t, tc) + + em := envloader.New().GetVarsFromEnv() + must.MapLen(t, 1, em) + must.MapContainsKey(t, em, mapKey) + + tV := em[mapKey] + actualErr := tc.inputParser.parseEnvVariable(getEnvKey(tc), tV) + + if tc.expectedError { + t.Logf(actualErr.Error()) + must.NotNil(t, actualErr) + return + } + must.Nil(t, actualErr, must.Sprintf("actualErr: %v", actualErr)) + must.MapEq(t, tc.expectedFlagVars, tc.inputParser.flagOverrideVars) + must.MapEq(t, tc.expectedEnvVars, tc.inputParser.envOverrideVars) + }) + } +} + +func TestParserV2_parseHeredocAtEOF(t *testing.T) { + inputParser := &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ + RootVariableFiles: map[pack.ID]*pack.File{}, + }, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{}, + fileOverrideVars: make(variables.PackIDKeyedVarMap), + } + + fixtureRoot := testfixture.AbsPath(t, "v2/variable_test") + p, err := loader.Load(fixtureRoot + "/variable_test") + must.NoError(t, err) + must.NotNil(t, p) + + inputParser.cfg.RootVariableFiles = p.RootVariableFiles() + + _, diags := inputParser.newParseOverridesFile(path.Join(fixtureRoot, "/heredoc.vars.hcl")) + must.False(t, diags.HasErrors(), must.Sprintf("diags: %v", diags)) + must.Len(t, 1, inputParser.fileOverrideVars["variable_test_pack"]) + must.Eq(t, "heredoc\n", inputParser.fileOverrideVars["variable_test_pack"][0].Value.AsString()) +} + +func TestParserV2_VariableOverrides(t *testing.T) { + testcases := []struct { + Name string + Parser *ParserV2 + Expect string + }{ + { + Name: "no override", + Parser: NewTestInputParserV2(), + Expect: "root", + }, + { + Name: "env override", + Parser: NewTestInputParserV2(WithEnvVar("input", "env")), + Expect: "env", + }, + { + Name: "file override", + Parser: NewTestInputParserV2(WithFileVar("input", "file")), + Expect: "file", + }, + { + Name: "flag override", + Parser: NewTestInputParserV2(WithCliVar("input", "flag")), + Expect: "flag", + }, + { + Name: "file opaques env", + Parser: NewTestInputParserV2( + WithEnvVar("input", "env"), + WithFileVar("input", "file"), + ), + Expect: "file", + }, + { + Name: "flag opaques env", + Parser: NewTestInputParserV2( + WithEnvVar("input", "env"), + WithCliVar("input", "flag"), + ), + Expect: "flag", + }, + { + Name: "flag opaques file", + Parser: NewTestInputParserV2( + WithFileVar("input", "file"), + WithCliVar("input", "flag"), + ), + Expect: "flag", + }, + { + Name: "flag opaques env and file", + Parser: NewTestInputParserV2( + WithEnvVar("input", "env"), + WithFileVar("input", "file"), + WithCliVar("input", "flag"), + ), + Expect: "flag", + }, + } + + for _, tc := range testcases { + t.Run(tc.Name, func(t *testing.T) { + pv, diags := tc.Parser.Parse() + must.NotNil(t, pv) + must.SliceEmpty(t, diags) + + must.Eq(t, tc.Expect, pv.v2Vars["example"]["input"].Value.AsString()) + }) + } +} + +type testParserV2Option func(*ParserV2) + +func WithEnvVar(key, value string) testParserV2Option { + return func(p *ParserV2) { + p.envOverrideVars["example"] = append(p.envOverrideVars["example"], NewStringVariableV2(key, value, "env")) + } +} + +func WithCliVar(key, value string) testParserV2Option { + return func(p *ParserV2) { + p.flagOverrideVars["example"] = append(p.flagOverrideVars["example"], NewStringVariableV2(key, value, "cli")) + } +} + +func WithFileVar(key, value string) testParserV2Option { + return func(p *ParserV2) { + p.flagOverrideVars["example"] = append(p.flagOverrideVars["example"], NewStringVariableV2(key, value, "file")) + } +} + +func NewTestInputParserV2(opts ...testParserV2Option) *ParserV2 { + + p := &ParserV2{ + fs: afero.Afero{Fs: afero.OsFs{}}, + cfg: &config.ParserConfig{ParentPackID: "example"}, + rootVars: map[pack.ID]map[variables.ID]*variables.Variable{ + "example": { + "input": &variables.Variable{ + Name: "input", + Type: cty.String, + Value: cty.StringVal("root"), + DeclRange: hcl.Range{Filename: ""}, + }, + }, + }, + envOverrideVars: make(variables.PackIDKeyedVarMap), + fileOverrideVars: make(variables.PackIDKeyedVarMap), + flagOverrideVars: make(variables.PackIDKeyedVarMap), + } + + // Loop through each option + for _, opt := range opts { + opt(p) + } + + return p +} + +func NewStringVariableV2(key, value, kind string) *variables.Variable { + return &variables.Variable{ + Name: variables.ID(key), + Type: cty.String, + Value: cty.StringVal(value), + DeclRange: hcl.Range{Filename: fmt.Sprintf("", key, kind)}, + } +} diff --git a/internal/pkg/variable/parser/template_context.go b/internal/pkg/variable/parser/template_context.go new file mode 100644 index 00000000..0fe5c1f0 --- /dev/null +++ b/internal/pkg/variable/parser/template_context.go @@ -0,0 +1,316 @@ +package parser + +import ( + "errors" + "fmt" + "slices" + "strings" + "text/template" + + "github.com/hashicorp/nomad-pack/sdk/pack" +) + +// PackTemplateContext v2 +// +// A v2 PackTemplateContext is a tree of PackData elements organized by name and +// dependency depth. Consider the following Pack structure: +// +// A Pack named "a", which will be the pack that is being targeted by the user +// via the nomad-pack command. The "a" pack's metadata points to two dependency +// packs: "d1" aliased to "c1" and "d2" aliased to "c2". Lastly, the "d2" pack's +// metadata points to a singular dependency named "dep", which is left unaliased. +// +// The generated template context would match the following diagram. +// +// ┌────────────────────────────────────────────────────────┐ +// │ PackTemplateContext │ +// │ map[string]PackContextable │ +// ├───────┬────────────────────────────────────────────────┤ +// │ KEY │ VALUE │ +// ├───────┼────────────────────────────────────────────────┤ +// │ │ ┌────────────────────┐ │ +// │ _self │ │PackData │ │ +// │ │ ├────────────────────┤ │ +// │ │ │Pack *pack.Pack │ │ +// │ │ │meta map[string]any │ │ +// │ │ │vars map[string]any │ │ +// │ │ └────────────────────┘ │ +// ├───────┼────────────────────────────────────────────────┤ +// │ │ ┌────────────────────────────────┐ │ +// │ c1 │ │ PackTemplateContext │ │ +// │ │ │ map[string]PackContextable │ │ +// │ │ ├───────┬────────────────────────┤ │ +// │ │ │ KEY │ VALUE │ │ +// │ │ ├───────┼────────────────────────┤ │ +// │ │ │ │ ┌────────────────────┐ │ │ +// │ │ │ _self │ │PackData │ │ │ +// │ │ │ │ ├────────────────────┤ │ │ +// │ │ │ │ │Pack *pack.Pack │ │ │ +// │ │ │ │ │meta map[string]any │ │ │ +// │ │ │ │ │vars map[string]any │ │ │ +// │ │ │ │ └────────────────────┘ │ │ +// │ │ └───────┴────────────────────────┘ │ +// ├───────┼────────────────────────────────────────────────┤ +// │ │ ┌────────────────────────────────────────────┐ │ +// │ c2 │ │ PackTemplateContext │ │ +// │ │ │ map[string]PackContextable │ │ +// │ │ ├───────┬────────────────────────────────────┤ │ +// │ │ │ KEY │ VALUE │ │ +// │ │ ├───────┼────────────────────────────────────┤ │ +// │ │ │ │ ┌────────────────────┐ │ │ +// │ │ │ _self │ │PackData │ │ │ +// │ │ │ │ ├────────────────────┤ │ │ +// │ │ │ │ │Pack *pack.Pack │ │ │ +// │ │ │ │ │meta map[string]any │ │ │ +// │ │ │ │ │vars map[string]any │ │ │ +// │ │ │ │ └────────────────────┘ │ │ +// │ │ ├───────┼────────────────────────────────────┤ │ +// │ │ │ │ ┌────────────────────────────────┐ │ │ +// │ │ │ dep │ │ PackTemplateContext │ │ │ +// │ │ │ │ │ map[string]PackContextable │ │ │ +// │ │ │ │ ├───────┬────────────────────────┤ │ │ +// │ │ │ │ │ KEY │ VALUE │ │ │ +// │ │ │ │ ├───────┼────────────────────────┤ │ │ +// │ │ │ │ │ │ ┌────────────────────┐ │ │ │ +// │ │ │ │ │ _self │ │PackData │ │ │ │ +// │ │ │ │ │ │ ├────────────────────┤ │ │ │ +// │ │ │ │ │ │ │Pack *pack.Pack │ │ │ │ +// │ │ │ │ │ │ │meta map[string]any │ │ │ │ +// │ │ │ │ │ │ │vars map[string]any │ │ │ │ +// │ │ │ │ │ │ └────────────────────┘ │ │ │ +// │ │ │ │ └───────┴────────────────────────┘ │ │ +// │ │ └───────┴────────────────────────────────────┘ │ +// └───────┴────────────────────────────────────────────────┘ +// +// This organization scheme allows users to select a pack using a dotted descent +// into the context map and to access the pack's data using functions that then +// read the PackData at `_self` + +type PackTemplateContext map[string]PackContextable + +type PackContextable interface { + getPack() PackData + getVars() map[string]any + getMetas() map[string]any +} + +const CurrentPackKey = "_self" + +// getVars retrieves the `vars` map of the PackData at the `CurrentPackKey` key +func (p PackTemplateContext) getVars() map[string]any { return p.getPack().vars } + +// getMetas retrieves the `meta` map of the PackData at the `CurrentPackKey` key +func (p PackTemplateContext) getMetas() map[string]any { return p.getPack().meta } + +// getPack retrieves the PackData at the `CurrentPackKey` key +func (p PackTemplateContext) getPack() PackData { return p[CurrentPackKey].(PackData) } + +// PackData is the currently selected Pack's metadata and variables, normally +// stored at `CurrentPackKey` in a PackTemplateContext. +type PackData struct { + Pack *pack.Pack + meta map[string]any + vars map[string]any +} + +// getVars returns the vars value from a PackData +func (p PackData) getVars() map[string]any { return p.vars } + +// getMetas returns the meta value from a PackData +func (p PackData) getMetas() map[string]any { return p.meta } + +// getPack returns this PackData +func (p PackData) getPack() PackData { return p } + +// +// Template function implementations +// + +// PackTemplateContextFuncs returns a text/template FuncMap that are necessary +// to access the variables and metadata information stored in the template +// context +func PackTemplateContextFuncs(isV1 bool) template.FuncMap { + if isV1 { + return PackTemplateContextFuncsV1() + } + return PackTemplateContextFuncsV2() +} + +// PackTemplateContextFuncsV1 returns the a funcMap with error-only functions +// with the same names as v2 ones, so users are presented with more informative +// errors than the generic go-template ones. +func PackTemplateContextFuncsV1() template.FuncMap { + fm := PackTemplateContextFuncsV2() + for k := range fm { + k := k + fm[k] = func(_ ...any) (string, error) { + return "", fmt.Errorf("%s is not implemented for nomad-pack's v1 syntax", k) + } + } + return fm +} + +// PackTemplateContextFuncsV2 returns the funcMap for the V2 Pack template +// context. These are added to other template functions provided in the +// Renderer +func PackTemplateContextFuncsV2() template.FuncMap { + return template.FuncMap{ + "vars": getPackVars, + "var": getPackVar, + "must_var": mustGetPackVar, + "metas": getPackMetas, + "meta": getPackMeta, + "must_meta": mustGetPackMeta, + "deps": getPackDeps, + "deps_tree": getPackDepTree, + } +} + +// getPackVars is the underlying implementation for the `vars` template func +func getPackVars(p PackContextable) map[string]any { return p.getVars() } + +// getPackVar is the underlying implementation for the `var` template func +func getPackVar(k string, p PackContextable) any { + if v, err := mustGetPackVar(k, p); err == nil { + return v + } else { + return "" + } +} + +// mustGetPackVar is the underlying implementation for the `must_var` template +// func +func mustGetPackVar(k string, p PackContextable) (any, error) { + return mustGetPackVarR(strings.Split(k, "."), p.getVars()) +} + +// mustGetPackMetaR recursively descends into a pack's variable map to collect +// the values. +func mustGetPackVarR(keys []string, p map[string]any) (any, error) { + if len(keys) > 0 { + np, found := p[keys[0]] + if !found { + // TODO: This should probably be the full traversal to this point accumulated. + return nil, fmt.Errorf("var key %s not found", keys[0]) + } + + if found && len(keys) == 1 { + return np, nil + } + + // If we're here, there's more than one key remaining in the traversal. + // See if we can continue + switch item := np.(type) { + case string: + return nil, errors.New("encountered non-traversable key while traversing") + + case map[string]any: + return mustGetPackVarR(keys[1:], item) + } + } + + return nil, errors.New("var key not found") +} + +// getPackMetas is the underlying implementation for the `metas` template func +func getPackMetas(p PackContextable) map[string]any { return p.getMetas() } + +// mustGetPackMeta is the underlying implementation for the `must_meta` template +// func +func mustGetPackMeta(k string, p PackContextable) (any, error) { + return mustGetPackMetaR(strings.Split(k, "."), p.getMetas()) +} + +// mustGetPackMetaR recursively descends into a pack's metadata map to collect +// the values. +func mustGetPackMetaR(keys []string, p map[string]any) (any, error) { + if len(keys) == 0 { + return nil, errors.New("end of traversal") + } + np, found := p[keys[0]] + if !found { + return nil, fmt.Errorf("meta key %s not found", keys[0]) + } + + switch item := np.(type) { + case string: + if len(keys) == 1 { + return item, nil + } + return nil, errors.New("encountered non-map key while traversing") + case map[string]any: + if len(keys) == 1 { + return nil, errors.New("traversal ended on non-metadata item key") + } + return mustGetPackMetaR(keys[1:], item) + default: + return nil, fmt.Errorf("meta key not found and hit non-traversible type (%T)", np) + } +} + +// getPackMeta is the underlying implementation for the `meta` template func +func getPackMeta(k string, p PackContextable) any { + if v, err := mustGetPackMeta(k, p); err == nil { + return v + } else { + return "" + } +} + +func getPackDeps(p PackTemplateContext) PackTemplateContext { + out := make(PackTemplateContext, len(p)-1) + for k, v := range p { + if k != CurrentPackKey { + out[k] = v + } + } + return out +} + +func getPackDepTree(p PackTemplateContext) []string { + if len(p) <= 1 { + return []string{} + } + + pAcc := new([]string) + + for _, k := range p.depKeys() { + v := p[k] + path := "." + k + *pAcc = append(*pAcc, path) + ptc := v.(PackTemplateContext) + getPackDepTreeR(ptc, path, pAcc) + + } + return *pAcc +} + +func getPackDepTreeR(p PackTemplateContext, path string, pAcc *[]string) { + if len(p) <= 1 { + return + } + + for _, k := range p.depKeys() { + v := p[k] + path = path + "." + k + *pAcc = append(*pAcc, path) + ptc := v.(PackTemplateContext) + getPackDepTreeR(ptc, path, pAcc) + } +} + +func (p PackTemplateContext) depKeys() []string { + out := make([]string, 0, len(p)-1) + for k := range p { + if k == CurrentPackKey { + continue + } + out = append(out, k) + } + slices.Sort(out) + return out +} + +func (p PackTemplateContext) Name() string { + return p.getPack().Pack.Name() +} diff --git a/internal/pkg/variable/root.go b/internal/pkg/variable/root.go deleted file mode 100644 index efebf722..00000000 --- a/internal/pkg/variable/root.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "github.com/hashicorp/hcl/v2" -) - -func (p *Parser) parseRootFiles() hcl.Diagnostics { - - var diags hcl.Diagnostics - - // Iterate all our root variable files. - for name, file := range p.cfg.RootVariableFiles { - - hclBody, loadDiags := p.loadPackFile(file) - diags = safeDiagnosticsExtend(diags, loadDiags) - - content, contentDiags := hclBody.Content(variableFileSchema) - diags = safeDiagnosticsExtend(diags, contentDiags) - - rootVars, parseDiags := p.parseRootBodyContent(content) - diags = safeDiagnosticsExtend(diags, parseDiags) - - // If we don't have any errors processing the file, and it's content, - // add an entry. - if !diags.HasErrors() { - p.rootVars[name] = rootVars - } - } - - return diags -} - -// parseRootBodyContent process the body of a root variables file, parsing -// each variable block found. -func (p *Parser) parseRootBodyContent(body *hcl.BodyContent) (map[string]*Variable, hcl.Diagnostics) { - - packRootVars := map[string]*Variable{} - - var diags hcl.Diagnostics - - // Due to the parsing that uses variableFileSchema, it is safe to assume - // every block has a type "variable". - for _, block := range body.Blocks { - cfg, cfgDiags := decodeVariableBlock(block) - diags = safeDiagnosticsExtend(diags, cfgDiags) - if cfg != nil { - packRootVars[cfg.Name] = cfg - } - } - return packRootVars, diags -} diff --git a/internal/pkg/variable/schema.go b/internal/pkg/variable/schema.go deleted file mode 100644 index d122d906..00000000 --- a/internal/pkg/variable/schema.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import "github.com/hashicorp/hcl/v2" - -const ( - variableAttributeType = "type" - variableAttributeDefault = "default" - variableAttributeDescription = "description" -) - -// variableFileSchema defines the hcl.BlockHeaderSchema for each root variable -// block. It allows us to capture the label for use as the variable name. -var variableFileSchema = &hcl.BodySchema{ - Blocks: []hcl.BlockHeaderSchema{ - { - Type: "variable", - LabelNames: []string{"name"}, - }, - }, -} - -// variableBlockSchema defines the hcl.BodySchema for a root variable block. It -// allows us to decode blocks effectively. -var variableBlockSchema = &hcl.BodySchema{ - Attributes: []hcl.AttributeSchema{ - {Name: variableAttributeDescription}, - {Name: variableAttributeDefault}, - {Name: variableAttributeType}, - }, -} diff --git a/internal/pkg/variable/schema/schema.go b/internal/pkg/variable/schema/schema.go new file mode 100644 index 00000000..e4c4d1a9 --- /dev/null +++ b/internal/pkg/variable/schema/schema.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package schema + +import "github.com/hashicorp/hcl/v2" + +const ( + VariableAttributeType = "type" + VariableAttributeDefault = "default" + VariableAttributeDescription = "description" +) + +// VariableFileSchema defines the hcl.BlockHeaderSchema for each root variable +// block. It allows us to capture the label for use as the variable name. +var VariableFileSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "variable", + LabelNames: []string{"name"}, + }, + }, +} + +// VariableBlockSchema defines the hcl.BodySchema for a root variable block. It +// allows us to decode blocks effectively. +var VariableBlockSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + {Name: VariableAttributeDescription}, + {Name: VariableAttributeDefault}, + {Name: VariableAttributeType}, + }, +} diff --git a/internal/pkg/variable/variable.go b/internal/pkg/variable/variable.go deleted file mode 100644 index f9c743e2..00000000 --- a/internal/pkg/variable/variable.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package variable - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/nomad-pack/sdk/pack" - "github.com/mitchellh/go-wordwrap" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" -) - -const VarEnvPrefix = "NOMAD_PACK_VAR_" - -// Variable encapsulates a single variable as defined within a block according -// to variableFileSchema and variableBlockSchema. -type Variable struct { - - // Name is the variable label. This is used to identify variables being - // overridden and during templating. - Name string - - // Description is an optional field which provides additional context to - // users identifying what the variable is used for. - Description string - hasDescription bool - - // Default is an optional field which provides a default value to be used - // in the absence of a user-provided value. It is only in this struct for - // documentation purposes - Default cty.Value - hasDefault bool - - // Type represents the concrete cty type of this variable. If the type is - // unable to be parsed into a cty type, it is invalid. - Type cty.Type - hasType bool - - // Value stores the variable value and is used when converting the cty type - // value into a Go type value. - Value cty.Value - - // DeclRange is the position marker of the variable within the file it was - // read from. This is used for diagnostics. - DeclRange hcl.Range -} - -func (v *Variable) merge(new *Variable) hcl.Diagnostics { - var diags hcl.Diagnostics - if new.Default != cty.NilVal { - v.hasDefault = new.hasDefault - v.Default = new.Default - } - - if new.Value != cty.NilVal { - v.Value = new.Value - } - - if new.Type != cty.NilType { - v.hasType = new.hasType - v.Type = new.Type - } - - if v.Value != cty.NilVal { - val, err := convert.Convert(v.Value, v.Type) - if err != nil { - switch { - case new.Type != cty.NilType && new.Value == cty.NilVal: - diags = safeDiagnosticsAppend(diags, - diagnosticInvalidDefaultValue( - fmt.Sprintf("Overriding this variable's type constraint has made its default value invalid: %s.", err), - new.DeclRange.Ptr(), - )) - case new.Type == cty.NilType && new.Value != cty.NilVal: - diags = safeDiagnosticsAppend(diags, - diagnosticInvalidDefaultValue( - fmt.Sprintf("The overridden default value for this variable is not compatible with the variable's type constraint: %s.", err), - new.DeclRange.Ptr(), - )) - default: - diags = safeDiagnosticsAppend(diags, - diagnosticInvalidDefaultValue( - fmt.Sprintf("This variable's default value is not compatible with its type constraint: %s.", err), - new.DeclRange.Ptr(), - )) - } - } else { - v.Value = val - } - } - - return diags -} - -// ParsedVariables wraps the parsed variables returned by parser.Parse and -// provides functionality to access them. -type ParsedVariables struct { - Vars map[string]map[string]*Variable - Metadata *pack.Metadata -} - -// ConvertVariablesToMapInterface translates the parsed variables into their -// native go types. The returned map is always keyed by the pack namespace for -// the variables. -// -// Even though parsing the variable went without error, it is highly possible -// that conversion to native go types can incur an error. If an error is -// returned, it should be considered terminal. -func (p *ParsedVariables) ConvertVariablesToMapInterface() (map[string]any, hcl.Diagnostics) { - - // Create our output; no matter what we return something. - out := make(map[string]any) - - // Errors can occur when performing the translation. We want to capture all - // of these and return them to the user. This allows them to fix problems - // in a single cycle. - var diags hcl.Diagnostics - - // Iterate each set of pack variable. - for packName, variables := range p.Vars { - - // packVar collects all variables associated to a pack. - packVar := map[string]any{} - - // Convert each variable and add this to the pack map. - for variableName, variable := range variables { - varInterface, err := convertCtyToInterface(variable.Value) - if err != nil { - diags = safeDiagnosticsAppend(diags, diagnosticFailedToConvertCty(err, variable.DeclRange.Ptr())) - continue - } - packVar[variableName] = varInterface - } - - // Add the pack variable to the full output. - out[packName] = packVar - } - - return out, diags -} - -func (v Variable) String() string { return asJSON(v) } -func (vf ParsedVariables) String() string { return asJSON(vf) } - -func asJSON(a any) string { - return func() string { b, _ := json.MarshalIndent(a, "", " "); return string(b) }() -} - -func (v Variable) AsOverrideString(packName string) string { - var out strings.Builder - out.WriteString(fmt.Sprintf(`# variable "%s.%s"`, packName, v.Name)) - out.WriteByte('\n') - if v.hasDescription { - tmp := "description: " + v.Description - wrapped := wordwrap.WrapString(tmp, 80) - lines := strings.Split(wrapped, "\n") - for i, l := range lines { - lines[i] = "# " + l - } - wrapped = strings.Join(lines, "\n") - out.WriteString(wrapped) - out.WriteString("\n") - } - if v.hasType { - out.WriteString(fmt.Sprintf("# type: %s\n", printType(v.Type))) - } - - if v.hasDefault { - out.WriteString(fmt.Sprintf("# default: %s\n", printDefault(v.Value))) - } - if v.hasDefault { - out.WriteString(fmt.Sprintf("#\n# %s.%s=%s\n\n", packName, v.Name, printDefault(v.Value))) - } - out.WriteString("\n") - return out.String() -} - -func (vf ParsedVariables) AsOverrideFile() string { - var out strings.Builder - out.WriteString(vf.varFileHeader()) - // TODO: this should have a stable order. - for p, vs := range vf.Vars { - for _, v := range vs { - out.WriteString(v.AsOverrideString(p)) - } - } - - return out.String() -} - -func (vf ParsedVariables) varFileHeader() string { - // Use pack metadata to enhance the header if desired. - // _ = vf.Metadata - // This value will be added to the top of the varfile - return "" -} - -// printType recursively prints out a cty.Type specification in a format that -// matched the way in which it is defined. -func printType(t cty.Type) string { - return printTypeR(t) -} - -func printTypeR(t cty.Type) string { - switch { - case t.IsPrimitiveType(): - return t.FriendlyNameForConstraint() - case t.IsListType(): - return "list(" + printTypeR(t.ElementType()) + ")" - case t.IsMapType(): - return "map(" + printTypeR(t.ElementType()) + ")" - case t.IsSetType(): - return "set(" + printTypeR(t.ElementType()) + ")" - case t.IsTupleType(): - tts := t.TupleElementTypes() - tfts := make([]string, len(tts)) - for i, tt := range tts { - if tt.IsPrimitiveType() { - tfts[i] = tt.FriendlyNameForConstraint() - } else { - tfts[i] = printTypeR(tt) - } - } - return "tuple(" + strings.Join(tfts, ", ") + ")" - case t.IsObjectType(): - at := t.AttributeTypes() - ats := make([]string, len(at)) - i := 0 - for n, a := range at { - if a.IsPrimitiveType() { - ats[i] = n + " = " + a.FriendlyNameForConstraint() - } else { - ats[i] = n + " = " + printTypeR(a) - } - i++ - } - return "object({" + strings.Join(ats, ", ") + "})" - default: - return "«unknown type»" - } -} - -// printDefault recursively prints out a cty.Value specification in a format -// that matched the way it is defined. This allows us to not have to capture -// or replicate the original presentation. However, could this be captured in -// parsing? -func printDefault(v cty.Value) string { - return printDefaultR(v) -} - -func printDefaultR(v cty.Value) string { - t := v.Type() - switch { - case t.IsPrimitiveType(): - return printPrimitiveValue(v) - - case t.IsListType(), t.IsSetType(), t.IsTupleType(): - // TODO, these could be optimized to be non-recursive calls for lists and sets of non-collection type - acc := make([]string, 0, v.LengthInt()) - v.ForEachElement(func(key cty.Value, val cty.Value) bool { acc = append(acc, printDefaultR(val)); return false }) - return "[" + strings.Join(acc, ", ") + "]" - - case t.IsMapType(), t.IsObjectType(): - acc := make([]string, 0, v.LengthInt()) - v.ForEachElement( - func(key cty.Value, val cty.Value) bool { - acc = append(acc, fmt.Sprintf("%s = %s", printDefaultR(key), printDefaultR(val))) - return false - }, - ) - return "{" + strings.Join(acc, ", ") + "}" - default: - return "«unknown value type»" - } -} - -func printPrimitiveValue(v cty.Value) string { - vI, _ := convertCtyToInterface(v) - if v.Type() == cty.String { - return fmt.Sprintf("%q", vI) - } - return fmt.Sprintf("%v", vI) -} diff --git a/sdk/pack/dependency.go b/sdk/pack/dependency.go index 46aa4018..32583689 100644 --- a/sdk/pack/dependency.go +++ b/sdk/pack/dependency.go @@ -15,6 +15,11 @@ type Dependency struct { // a dependency with different variables. Name string `hcl:"name,label"` + // Alias overrides the dependency pack's Name in references when set, + // allowing the same pack source to be used multiple times as with different + // variable values. + Alias string `hcl:"alias,optional"` + // Source is the remote source where the pack can be fetched. This string // can follow any format as supported by go-getter or be a local path // indicating the pack has already been downloaded. @@ -25,6 +30,21 @@ type Dependency struct { Enabled *bool `hcl:"enabled,optional"` } +// AliasOrName returns the pack's Alias or the pack's Name, preferring the +// Alias when set. +func (d *Dependency) AliasOrName() string { + if d.Alias != "" { + return d.Alias + } + return d.Name +} + +// ID returns the identifier for the pack. The function returns a ID +// which implements the Stringer interface +func (d *Dependency) ID() ID { + return ID(d.AliasOrName()) +} + // validate the Dependency object to ensure it meets requirements and doesn't // contain invalid or incorrect data. func (d *Dependency) validate() error { diff --git a/sdk/pack/dependency_test.go b/sdk/pack/dependency_test.go index 7ac4436a..158e6670 100644 --- a/sdk/pack/dependency_test.go +++ b/sdk/pack/dependency_test.go @@ -57,12 +57,15 @@ func TestDependency_Validate(t *testing.T) { } for _, tc := range testCases { - err := tc.inputDependency.validate() - if tc.expectError { - must.NotNil(t, err, must.Sprint(tc.name)) - } else { - must.Nil(t, err, must.Sprint(tc.name)) - } - must.Eq(t, tc.expectedOutputDependency, tc.inputDependency, must.Sprint(tc.name)) + t.Run(tc.name, func(t *testing.T) { + tc := tc + err := tc.inputDependency.validate() + if tc.expectError { + must.NotNil(t, err) + } else { + must.Nil(t, err) + } + must.Eq(t, tc.expectedOutputDependency, tc.inputDependency) + }) } } diff --git a/sdk/pack/metadata.go b/sdk/pack/metadata.go index ea4bd27f..6beae76d 100644 --- a/sdk/pack/metadata.go +++ b/sdk/pack/metadata.go @@ -43,6 +43,9 @@ type MetadataPack struct { // rendering. Name string `hcl:"name"` + // Alias will optionally override the provided Pack name value when set + Alias string `hcl:"alias,optional"` + // Description is a small overview of the application that is deployed by // the pack. Description string `hcl:"description,optional"` @@ -91,7 +94,7 @@ type MetadataIntegration struct { // metadata object. The conversion doesn't take into account empty values and // will add them. func (md *Metadata) ConvertToMapInterface() map[string]any { - innerMap := map[string]any{ + m := map[string]any{ "app": map[string]any{ "url": md.App.URL, }, @@ -100,40 +103,30 @@ func (md *Metadata) ConvertToMapInterface() map[string]any { "description": md.Pack.Description, "version": md.Pack.Version, }, + "dependencies": []map[string]any{}, } if md.Integration != nil { - innerMap["integration"] = map[string]any{ + m["integration"] = map[string]any{ "identifier": md.Integration.Identifier, "flags": md.Integration.Flags, "name": md.Integration.Name, } } - return map[string]any{"nomad_pack": innerMap} -} - -// AddToInterfaceMap adds the metadata information to the provided map as a new -// entry under the "nomad_pack" key. This is useful for adding this information -// to the template rendering data. -func (md *Metadata) AddToInterfaceMap(m map[string]any) map[string]any { - innerMap := map[string]any{ - "app": map[string]any{ - "url": md.App.URL, - }, - "pack": map[string]any{ - "name": md.Pack.Name, - "description": md.Pack.Description, - "version": md.Pack.Version, - }, - } - if md.Integration != nil { - innerMap["integration"] = map[string]any{ - "identifier": md.Integration.Identifier, - "flags": md.Integration.Flags, - "name": md.Integration.Name, + dSlice := make([]map[string]any, len(md.Dependencies)) + for i, d := range md.Dependencies { + dSlice[i] = map[string]any{ + d.AliasOrName(): map[string]any{ + "id": d.AliasOrName(), + "name": d.Name, + "alias": d.Alias, + "source": d.Source, + "enabled": d.Enabled, + }, } } - m["nomad_pack"] = innerMap + m["dependencies"] = dSlice + return m } @@ -178,3 +171,11 @@ func (mp *MetadataPack) validate() error { } return nil } + +// AddToInterfaceMap adds the metadata information to the provided map as a new +// entry under the "nomad_pack" key. This is useful for adding this information +// to the template rendering data. Used in the deprecated V1 Renderer +func (md *Metadata) AddToInterfaceMap(m map[string]any) map[string]any { + m["nomad_pack"] = md.ConvertToMapInterface() + return m +} diff --git a/sdk/pack/metadata_test.go b/sdk/pack/metadata_test.go index 25b8a0bd..d14d5d73 100644 --- a/sdk/pack/metadata_test.go +++ b/sdk/pack/metadata_test.go @@ -11,11 +11,12 @@ import ( func TestMetadata_ConvertToMapInterface(t *testing.T) { testCases := []struct { + name string inputMetadata *Metadata expectedOutput map[string]any - name string }{ { + name: "all metadata values populated", inputMetadata: &Metadata{ App: &MetadataApp{ URL: "https://example.com", @@ -28,35 +29,94 @@ func TestMetadata_ConvertToMapInterface(t *testing.T) { Integration: &MetadataIntegration{ Name: "Example", Identifier: "nomad/hashicorp/example", - Flags: []string{ - "foo", - "bar", - }, + Flags: []string{"foo", "bar"}, }, }, expectedOutput: map[string]any{ - "nomad_pack": map[string]any{ - "app": map[string]any{ - "url": "https://example.com", + "app": map[string]any{ + "url": "https://example.com", + }, + "pack": map[string]any{ + "name": "Example", + "description": "The most basic, yet awesome, example", + "version": "v0.0.1", + }, + "integration": map[string]any{ + "identifier": "nomad/hashicorp/example", + "flags": []string{"foo", "bar"}, + "name": "Example", + }, + "dependencies": []map[string]any{}, + }, + }, + { + name: "all metadata values with deps", + inputMetadata: &Metadata{ + App: &MetadataApp{ + URL: "https://example.com", + }, + Pack: &MetadataPack{ + Name: "Example", + Description: "The most basic, yet awesome, example", + Version: "v0.0.1", + }, + Integration: &MetadataIntegration{ + Name: "Example", + Identifier: "nomad/hashicorp/example", + Flags: []string{"foo", "bar"}, + }, + Dependencies: []*Dependency{ + { + Name: "dep1", + Enabled: pointerOf(true), }, - "pack": map[string]any{ - "name": "Example", - "description": "The most basic, yet awesome, example", - "version": "v0.0.1", + { + Name: "dep1", + Alias: "dep2", + Enabled: pointerOf(true), }, - "integration": map[string]any{ - "identifier": "nomad/hashicorp/example", - "flags": []string{ - "foo", - "bar", + }, + }, + expectedOutput: map[string]any{ + "app": map[string]any{ + "url": "https://example.com", + }, + "pack": map[string]any{ + "name": "Example", + "description": "The most basic, yet awesome, example", + "version": "v0.0.1", + }, + "integration": map[string]any{ + "identifier": "nomad/hashicorp/example", + "flags": []string{"foo", "bar"}, + "name": "Example", + }, + "dependencies": []map[string]any{ + { + "dep1": map[string]any{ + "name": "dep1", + "alias": "", + "id": "dep1", + "source": "", + "enabled": pointerOf(true), + }, + }, + { + "dep2": map[string]any{ + "name": "dep1", + "alias": "dep2", + "id": "dep2", + "source": "", + "enabled": pointerOf(true), }, - "name": "Example", }, }, }, - name: "all metadata values populated", }, { + // TODO test added to cover graceful failure while we're in the process of + // retiring "Author" and "URL" metadata fields. Can be removed in the future. + name: "author and url fields ignored gracefully", inputMetadata: &Metadata{ App: &MetadataApp{ URL: "https://example.com", @@ -69,25 +129,24 @@ func TestMetadata_ConvertToMapInterface(t *testing.T) { Integration: &MetadataIntegration{}, }, expectedOutput: map[string]any{ - "nomad_pack": map[string]any{ - "app": map[string]any{ - "url": "https://example.com", - }, - "pack": map[string]any{ - "name": "Example", - "description": "", - "version": "v0.0.1", - }, - "integration": map[string]any{ - "identifier": "", - "flags": []string(nil), - "name": "", - }, + "app": map[string]any{ + "url": "https://example.com", }, + "pack": map[string]any{ + "name": "Example", + "description": "", + "version": "v0.0.1", + }, + "integration": map[string]any{ + "identifier": "", + "flags": []string(nil), + "name": "", + }, + "dependencies": []map[string]any{}, }, - name: "some metadata values populated", }, { + name: "some metadata values populated", inputMetadata: &Metadata{ App: &MetadataApp{ URL: "https://example.com", @@ -99,21 +158,20 @@ func TestMetadata_ConvertToMapInterface(t *testing.T) { Integration: &MetadataIntegration{}, }, expectedOutput: map[string]any{ - "nomad_pack": map[string]any{ - "app": map[string]any{"url": "https://example.com"}, - "pack": map[string]any{"name": "", "description": "", "version": ""}, - "integration": map[string]any{"identifier": "", "flags": []string(nil), "name": ""}, - }, + "app": map[string]any{"url": "https://example.com"}, + "pack": map[string]any{"name": "", "description": "", "version": ""}, + "integration": map[string]any{"identifier": "", "flags": []string(nil), "name": ""}, + "dependencies": []map[string]any{}, }, - // TODO test added to cover graceful failure while we're in the process of - // retiring "Author" and "URL" metadata fields. Can be removed in the future. - name: "author and url fields ignored gracefully", }, } for _, tc := range testCases { - actualOutput := tc.inputMetadata.ConvertToMapInterface() - must.Eq(t, tc.expectedOutput, actualOutput, must.Sprint(tc.name)) + t.Run(tc.name, func(t *testing.T) { + tc := tc + actualOutput := tc.inputMetadata.ConvertToMapInterface() + must.Eq(t, tc.expectedOutput, actualOutput) + }) } } @@ -145,11 +203,14 @@ func TestMetadata_Validate(t *testing.T) { } for _, tc := range testCases { - err := tc.inputMetadata.Validate() - if tc.expectError { - must.NotNil(t, err, must.Sprint(tc.name)) - } else { - must.Nil(t, err, must.Sprint(tc.name)) - } + t.Run(tc.name, func(t *testing.T) { + tc := tc + err := tc.inputMetadata.Validate() + if tc.expectError { + must.NotNil(t, err, must.Sprint(tc.name)) + } else { + must.Nil(t, err, must.Sprint(tc.name)) + } + }) } } diff --git a/sdk/pack/pack.go b/sdk/pack/pack.go index f973cbbd..0227fab5 100644 --- a/sdk/pack/pack.go +++ b/sdk/pack/pack.go @@ -3,7 +3,21 @@ package pack -import "errors" +import ( + "errors" + "strings" +) + +type ID string + +func (p ID) String() string { return string(p) } + +// Join returns a new ID with the child path appended to it. +func (p ID) Join(child ID) ID { return ID(string(p) + "." + string(child)) } + +// AsPath returns a string with the dot delimiters converted to `/` for use with +// file system paths. +func (p ID) AsPath() string { return strings.ReplaceAll(string(p), ".", "/") } // File is an individual file component of a Pack. type File struct { @@ -53,18 +67,57 @@ type Pack struct { dependencies []*Pack // parent tracks the parent pack for dependencies. In the case that this is - // the parent pack, this will be nil. + // the root pack, this will be nil. parent *Pack + + // alias tracks the name assigned by the parent pack for any dependencies. + // In the case that this is the parent pack, this will be nil. + alias string } // Name returns the name of the pack. The canonical value for this comes from // the Pack.Name Metadata struct field. -func (p *Pack) Name() string { return p.Metadata.Pack.Name } +func (p *Pack) Name() string { + return p.Metadata.Pack.Name +} + +// Alias returns the alias assigned to the pack. The canonical value for this +// comes from the alias on a running pack with a fallback to the Pack.Alias +// Metadata struct field. +func (p *Pack) Alias() string { + if p.alias != "" { + return p.alias + } + return p.Metadata.Pack.Alias +} + +// AliasOrName returns the pack's Alias or the pack's Name, preferring the +// Alias when set. +func (p *Pack) AliasOrName() string { + if p.Alias() == "" { + return p.Name() + } + return p.Alias() +} + +// ID returns the identifier for the pack. The function returns a ID +// which implements the Stringer interface +func (p *Pack) ID() ID { + return ID(p.AliasOrName()) +} // HasParent reports whether this pack has a parent or can be considered the // top level pack. func (p *Pack) HasParent() bool { return p.parent != nil } +// AddDependency to the pack, correctly setting their parent pack identifier and +// alias. +func (p *Pack) AddDependency(alias ID, pack *Pack) { + pack.parent = p + pack.alias = alias.String() + p.dependencies = append(p.dependencies, pack) +} + // AddDependencies to the pack, correctly setting their parent pack identifier. func (p *Pack) AddDependencies(packs ...*Pack) { for i, depPack := range packs { @@ -73,26 +126,34 @@ func (p *Pack) AddDependencies(packs ...*Pack) { } } -// Dependencies returns the list of dependence the Pack has. +// Dependencies returns the list of dependencies the Pack has. func (p *Pack) Dependencies() []*Pack { return p.dependencies } // RootVariableFiles generates a mapping of all root variable files for the // pack and all dependencies. -func (p *Pack) RootVariableFiles() map[string]*File { +func (p *Pack) RootVariableFiles() map[ID]*File { // Set up the base output that include the top level packs root variable // file entry. - out := map[string]*File{p.Name(): p.RootVariableFile} + out := map[ID]*File{p.ID(): p.RootVariableFile} // Iterate the dependency packs and add entries into the variable file // mapping for each. for _, dep := range p.dependencies { - out[dep.Name()] = dep.RootVariableFile + dep.rootVariableFiles(p.ID(), &out) } return out } +func (p *Pack) rootVariableFiles(parentID ID, acc *map[ID]*File) { + depID := parentID.Join(p.ID()) + (*acc)[depID] = p.RootVariableFile + for _, dep := range p.dependencies { + dep.rootVariableFiles(depID, acc) + } +} + // Validate the pack for terminal problems that can easily be detected at this // stage. Anything that has potential to cause a panic should ideally be caught // here. @@ -108,3 +169,25 @@ func (p *Pack) Validate() error { return nil } + +func (p *Pack) VariablesPath() ID { + parts := variablesPathR(p, []string{}) + // Since variablesPathR is depth-first, we need + // to reverse it before joining it together + reverse(parts) + out := ID(strings.Join(parts, ".")) + return out +} + +func variablesPathR(p *Pack, in []string) []string { + if p.parent == nil { + return append(in, p.AliasOrName()) + } + return variablesPathR(p.parent, append(in, p.AliasOrName())) +} + +func reverse[T any](s []T) { + for first, last := 0, len(s)-1; first < last; first, last = first+1, last-1 { + s[first], s[last] = s[last], s[first] + } +} diff --git a/sdk/pack/pack_test.go b/sdk/pack/pack_test.go index f95fe7c7..d2367a76 100644 --- a/sdk/pack/pack_test.go +++ b/sdk/pack/pack_test.go @@ -6,10 +6,12 @@ package pack import ( "testing" + "github.com/hashicorp/nomad/ci" "github.com/shoenig/test/must" ) func TestPack_Name(t *testing.T) { + ci.Parallel(t) testCases := []struct { inputPack *Pack expectedOutput string @@ -28,14 +30,19 @@ func TestPack_Name(t *testing.T) { } for _, tc := range testCases { - must.Eq(t, tc.expectedOutput, tc.inputPack.Name(), must.Sprint(tc.name)) + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) // Parallel has to be called in the subtest too + must.Eq(t, tc.expectedOutput, tc.inputPack.Name()) + }) } } func TestPack_RootVariableFiles(t *testing.T) { + ci.Parallel(t) testCases := []struct { inputPack *Pack - expectedOutput map[string]*File + expectedOutput map[ID]*File name string }{ { @@ -51,7 +58,7 @@ func TestPack_RootVariableFiles(t *testing.T) { Content: []byte(`variable "foo" {default = "bar"}`), }, }, - expectedOutput: map[string]*File{ + expectedOutput: map[ID]*File{ "example": { Name: "variables.hcl", Path: "/opt/packs/example/variables.hcl", @@ -99,18 +106,18 @@ func TestPack_RootVariableFiles(t *testing.T) { }, }, }, - expectedOutput: map[string]*File{ + expectedOutput: map[ID]*File{ "example": { Name: "variables.hcl", Path: "/opt/packs/example/variables.hcl", Content: []byte(`variable "foo" {default = "bar"}`), }, - "dep1": { + "example.dep1": { Name: "variables.hcl", Path: "/opt/packs/dep1/variables.hcl", Content: []byte(`variable "hoo" {default = "har"}`), }, - "dep2": { + "example.dep2": { Name: "variables.hcl", Path: "/opt/packs/dep2/variables.hcl", Content: []byte(`variable "sun" {default = "start"}`), @@ -121,6 +128,10 @@ func TestPack_RootVariableFiles(t *testing.T) { } for _, tc := range testCases { - must.Eq(t, tc.expectedOutput, tc.inputPack.RootVariableFiles(), must.Sprint(tc.name)) + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) // Parallel has to be called in the subtest too + must.Eq(t, tc.expectedOutput, tc.inputPack.RootVariableFiles()) + }) } } diff --git a/internal/pkg/variable/convert.go b/sdk/pack/variables/convert.go similarity index 89% rename from internal/pkg/variable/convert.go rename to sdk/pack/variables/convert.go index 2cb5b91e..5298f7a7 100644 --- a/internal/pkg/variable/convert.go +++ b/sdk/pack/variables/convert.go @@ -1,9 +1,10 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -package variable +package variables import ( + "errors" "fmt" "math" "math/big" @@ -11,14 +12,14 @@ import ( "github.com/zclconf/go-cty/cty" ) -func convertCtyToInterface(val cty.Value) (any, error) { +func ConvertCtyToInterface(val cty.Value) (any, error) { if val.IsNull() { return nil, nil } if !val.IsKnown() { - return nil, fmt.Errorf("value is not known") + return nil, errors.New("value is not known") } t := val.Type() @@ -48,7 +49,7 @@ func convertCtyToInterface(val cty.Value) (any, error) { it := val.ElementIterator() for it.Next() { _, ev := it.Element() - evi, err := convertCtyToInterface(ev) + evi, err := ConvertCtyToInterface(ev) if err != nil { return nil, err } @@ -61,7 +62,7 @@ func convertCtyToInterface(val cty.Value) (any, error) { it := val.ElementIterator() for it.Next() { _, ev := it.Element() - evi, err := convertCtyToInterface(ev) + evi, err := ConvertCtyToInterface(ev) if err != nil { return nil, err } @@ -75,7 +76,7 @@ func convertCtyToInterface(val cty.Value) (any, error) { ek, ev := it.Element() ekv := ek.AsString() - evv, err := convertCtyToInterface(ev) + evv, err := ConvertCtyToInterface(ev) if err != nil { return nil, err } @@ -88,7 +89,7 @@ func convertCtyToInterface(val cty.Value) (any, error) { for k := range t.AttributeTypes() { av := val.GetAttr(k) - avv, err := convertCtyToInterface(av) + avv, err := ConvertCtyToInterface(av) if err != nil { return nil, err } diff --git a/internal/pkg/variable/convert_test.go b/sdk/pack/variables/convert_test.go similarity index 55% rename from internal/pkg/variable/convert_test.go rename to sdk/pack/variables/convert_test.go index a5efee99..abe550a6 100644 --- a/internal/pkg/variable/convert_test.go +++ b/sdk/pack/variables/convert_test.go @@ -1,17 +1,19 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 -package variable +package variables import ( "reflect" "testing" + "github.com/hashicorp/nomad/ci" "github.com/shoenig/test/must" "github.com/zclconf/go-cty/cty" ) func TestConvertCtyToInterface(t *testing.T) { + ci.Parallel(t) // test basic type testCases := []struct { @@ -26,8 +28,10 @@ func TestConvertCtyToInterface(t *testing.T) { } for _, tc := range testCases { + tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { - res, err := convertCtyToInterface(tc.val) + ci.Parallel(t) // Parallel has to be set on the subtest also + res, err := ConvertCtyToInterface(tc.val) must.NoError(t, err) resType := reflect.TypeOf(res).Kind() @@ -37,13 +41,14 @@ func TestConvertCtyToInterface(t *testing.T) { // test list of list t.Run("lists of lists", func(t *testing.T) { + ci.Parallel(t) // Parallel has to be set on the subtest also testListOfList := cty.ListVal([]cty.Value{ cty.ListVal([]cty.Value{ cty.BoolVal(true), }), }) - resListOfList, err := convertCtyToInterface(testListOfList) + resListOfList, err := ConvertCtyToInterface(testListOfList) must.NoError(t, err) tempList, ok := resListOfList.([]any) @@ -55,13 +60,14 @@ func TestConvertCtyToInterface(t *testing.T) { // test list of maps t.Run("list of maps", func(t *testing.T) { + ci.Parallel(t) // Parallel has to be set on the subtest also testListOfMaps := cty.ListVal([]cty.Value{ cty.MapVal(map[string]cty.Value{ "test": cty.BoolVal(true), }), }) - resListOfMaps, err := convertCtyToInterface(testListOfMaps) + resListOfMaps, err := ConvertCtyToInterface(testListOfMaps) must.NoError(t, err) _, ok := resListOfMaps.([]map[string]any) @@ -70,11 +76,12 @@ func TestConvertCtyToInterface(t *testing.T) { // test map of maps t.Run("map of maps", func(t *testing.T) { + ci.Parallel(t) // Parallel has to be set on the subtest also testMapOfMaps := cty.MapVal(map[string]cty.Value{ "test": cty.MapVal(map[string]cty.Value{"test": cty.BoolVal(true)}), }) - restMapOfMaps, err := convertCtyToInterface(testMapOfMaps) + restMapOfMaps, err := ConvertCtyToInterface(testMapOfMaps) must.NoError(t, err) tempMapOfMaps, ok := restMapOfMaps.(map[string]any) @@ -83,4 +90,31 @@ func TestConvertCtyToInterface(t *testing.T) { _, ok = tempMapOfMaps["test"].(map[string]any) must.True(t, ok) }) + + // test map of objects + t.Run("map of objects", func(t *testing.T) { + ci.Parallel(t) // Parallel has to be set on the subtest also + testMapOfObj := cty.MapVal(map[string]cty.Value{ + "t1": cty.ObjectVal(map[string]cty.Value{"b": cty.BoolVal(true)}), + "t2": cty.ObjectVal(map[string]cty.Value{"b": cty.BoolVal(false)}), + }) + + restMapOfObj, err := ConvertCtyToInterface(testMapOfObj) + must.NoError(t, err) + + tempMapOfObj, ok := restMapOfObj.(map[string]any) + must.True(t, ok) + + tp1, ok := tempMapOfObj["t1"].(map[string]any) + must.True(t, ok) + tp2, ok := tempMapOfObj["t2"].(map[string]any) + must.True(t, ok) + + b1, ok := tp1["b"].(bool) + must.True(t, ok) + must.True(t, b1) + b2, ok := tp2["b"].(bool) + must.True(t, ok) + must.False(t, b2) + }) } diff --git a/sdk/pack/variables/formatters.go b/sdk/pack/variables/formatters.go new file mode 100644 index 00000000..74d798a6 --- /dev/null +++ b/sdk/pack/variables/formatters.go @@ -0,0 +1,97 @@ +package variables + +import ( + "fmt" + "strings" + + "github.com/zclconf/go-cty/cty" +) + +// printType recursively prints out a cty.Type specification in a format that +// matched the way in which it is defined. +func printType(t cty.Type) string { + return printTypeR(t) +} + +func printTypeR(t cty.Type) string { + switch { + case t.IsPrimitiveType(): + return t.FriendlyNameForConstraint() + case t.IsListType(): + return "list(" + printTypeR(t.ElementType()) + ")" + case t.IsMapType(): + return "map(" + printTypeR(t.ElementType()) + ")" + case t.IsSetType(): + return "set(" + printTypeR(t.ElementType()) + ")" + case t.IsTupleType(): + tts := t.TupleElementTypes() + tfts := make([]string, len(tts)) + for i, tt := range tts { + if tt.IsPrimitiveType() { + tfts[i] = tt.FriendlyNameForConstraint() + } else { + tfts[i] = printTypeR(tt) + } + } + return "tuple(" + strings.Join(tfts, ", ") + ")" + case t.IsObjectType(): + at := t.AttributeTypes() + ats := make([]string, len(at)) + i := 0 + for n, a := range at { + if a.IsPrimitiveType() { + ats[i] = n + " = " + a.FriendlyNameForConstraint() + } else { + ats[i] = n + " = " + printTypeR(a) + } + i++ + } + return "object({" + strings.Join(ats, ", ") + "})" + case t.HasDynamicTypes(): + return ("dynamic") + default: + return "«unknown type»" + } +} + +// printDefault recursively prints out a cty.Value specification in a format +// that matched the way it is defined. This allows us to not have to capture +// or replicate the original presentation. However, could this be captured in +// parsing? +func printDefault(v cty.Value) string { + return printDefaultR(v) +} + +func printDefaultR(v cty.Value) string { + t := v.Type() + switch { + case t.IsPrimitiveType(): + return printPrimitiveValue(v) + + case t.IsListType(), t.IsSetType(), t.IsTupleType(): + // TODO, these could be optimized to be non-recursive calls for lists and sets of non-collection type + acc := make([]string, 0, v.LengthInt()) + v.ForEachElement(func(key cty.Value, val cty.Value) bool { acc = append(acc, printDefaultR(val)); return false }) + return "[" + strings.Join(acc, ", ") + "]" + + case t.IsMapType(), t.IsObjectType(): + acc := make([]string, 0, v.LengthInt()) + v.ForEachElement( + func(key cty.Value, val cty.Value) bool { + acc = append(acc, fmt.Sprintf("%s = %s", printDefaultR(key), printDefaultR(val))) + return false + }, + ) + return "{" + strings.Join(acc, ", ") + "}" + default: + return "«unknown value type»" + } +} + +func printPrimitiveValue(v cty.Value) string { + vI, _ := ConvertCtyToInterface(v) + if v.Type() == cty.String { + return fmt.Sprintf("%q", vI) + } + return fmt.Sprintf("%v", vI) +} diff --git a/sdk/pack/variables/formatters_test.go b/sdk/pack/variables/formatters_test.go new file mode 100644 index 00000000..991f4031 --- /dev/null +++ b/sdk/pack/variables/formatters_test.go @@ -0,0 +1,303 @@ +package variables + +import ( + "strings" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/shoenig/test/must" + "github.com/zclconf/go-cty/cty" +) + +func TestFormatters_printType(t *testing.T) { + ci.Parallel(t) + testCases := []struct { + name string + input cty.Type + expect string + err error + }{ + { + name: "string", + input: cty.String, + expect: "string", + }, + { + name: "bool", + input: cty.Bool, + expect: "bool", + }, + { + name: "number", + input: cty.Number, + expect: "number", + }, + { + name: "any", + input: cty.DynamicPseudoType, + expect: "dynamic", + }, + { + name: "list/string", + input: cty.List(cty.String), + expect: "list(string)", + }, + { + name: "list/bool", + input: cty.List(cty.Bool), + expect: "list(bool)", + }, + { + name: "list/number", + input: cty.List(cty.Number), + expect: "list(number)", + }, + { + name: "list/any", + input: cty.List(cty.DynamicPseudoType), + expect: "list(dynamic)", + }, + { + name: "map/string", + input: cty.Map(cty.String), + expect: "map(string)", + }, + { + name: "map/bool", + input: cty.Map(cty.Bool), + expect: "map(bool)", + }, + { + name: "map/number", + input: cty.Map(cty.Number), + expect: "map(number)", + }, + { + name: "map/any", + input: cty.Map(cty.DynamicPseudoType), + expect: "map(dynamic)", + }, + { + name: "set/string", + input: cty.Set(cty.String), + expect: "set(string)", + }, + { + name: "set/bool", + input: cty.Set(cty.Bool), + expect: "set(bool)", + }, + { + name: "set/number", + input: cty.Set(cty.Number), + expect: "set(number)", + }, + { + name: "set/any", + input: cty.Set(cty.DynamicPseudoType), + expect: "set(dynamic)", + }, + { + name: "tuple", + input: cty.Tuple([]cty.Type{cty.String, cty.Bool, cty.Number, cty.DynamicPseudoType}), + expect: "tuple(string, bool, number, dynamic)", + }, + } + for _, tc := range testCases { + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) + out := printType(tc.input) + must.Eq(t, tc.expect, out, must.Sprint(tc.input.FriendlyName())) + }) + } + + t.Run("object", func(t *testing.T) { + // Object is special cased because it has a map that prints in + // non-guaranteed order. + ci.Parallel(t) + var tc = struct { + name string + input cty.Type + expect string + err error + }{ + name: "object", + input: cty.Object(map[string]cty.Type{ + "str": cty.String, + "b": cty.Bool, + "num": cty.Number, + "any": cty.DynamicPseudoType}), + expect: "object({str = string, b = bool, num = number, any = dynamic})", + } + out := printType(tc.input) + exp := tc.expect + must.True(t, strings.HasPrefix(out, "object({")) + out = strings.TrimPrefix(out, "object({") + exp = strings.TrimPrefix(exp, "object({") + + must.True(t, strings.HasSuffix(out, "})")) + out = strings.TrimSuffix(out, "})") + exp = strings.TrimSuffix(exp, "})") + + oParts := strings.Split(out, ", ") + eParts := strings.Split(exp, ", ") + + must.SliceContainsAll(t, eParts, oParts) + }) +} + +func TestFormatters_printDefault(t *testing.T) { + ci.Parallel(t) + testCases := []struct { + name string + input cty.Value + expect string + err error + }{ + { + name: "string", + input: cty.StringVal("test"), + expect: `"test"`, + }, + { + name: "bool/true", + input: cty.BoolVal(true), + expect: "true", + }, + { + name: "bool/false", + input: cty.BoolVal(false), + expect: "false", + }, + { + name: "number/int", + input: cty.NumberIntVal(-100), + expect: "-100", + }, + { + name: "number/uint", + input: cty.NumberUIntVal(100), + expect: "100", + }, + { + name: "number/float/positive", + input: cty.NumberFloatVal(0.2), + expect: "0.2", + }, + { + name: "number/float/negative", + input: cty.NumberFloatVal(-0.2), + expect: "-0.2", + }, + { + name: "number/float/zero", + input: cty.NumberFloatVal(0), + expect: "0", + }, + { + name: "list/string", + input: cty.ListVal([]cty.Value{ + cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c"), + }), + expect: `["a", "b", "c"]`, + }, + { + name: "list/bool", + input: cty.ListVal([]cty.Value{ + cty.BoolVal(true), cty.BoolVal(false), cty.BoolVal(true), + }), + expect: `[true, false, true]`, + }, + { + name: "list/number", + input: cty.ListVal([]cty.Value{ + cty.NumberFloatVal(-1.1), + cty.NumberIntVal(-1), + cty.NumberIntVal(0), + cty.NumberUIntVal(0), + cty.NumberFloatVal(0), + cty.NumberIntVal(1), + cty.NumberFloatVal(1.1), + cty.NumberUIntVal(2), + }), + expect: `[-1.1, -1, 0, 0, 0, 1, 1.1, 2]`, + }, + { + name: "map/string", + input: cty.MapVal(map[string]cty.Value{ + "a": cty.StringVal("apple"), + "b": cty.StringVal("ball"), + }), + expect: `{"a" = "apple", "b" = "ball"}`, + }, + { + name: "map/bool", + input: cty.MapVal(map[string]cty.Value{ + "a": cty.BoolVal(false), + "b": cty.BoolVal(true), + }), + expect: `{"a" = false, "b" = true}`, + }, + { + name: "map/number", + input: cty.MapVal(map[string]cty.Value{ + "a": cty.NumberIntVal(0), + "b": cty.NumberFloatVal(2.4), + }), + expect: `{"a" = 0, "b" = 2.4}`, + }, + { + name: "set/string", + input: cty.SetVal([]cty.Value{ + cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c"), + }), + expect: `["a", "b", "c"]`, + }, + { + name: "set/bool", + input: cty.SetVal([]cty.Value{ + cty.BoolVal(true), cty.BoolVal(false), cty.BoolVal(true), + }), + expect: `[false, true]`, + }, + { + name: "set/number", + input: cty.SetVal([]cty.Value{ + cty.NumberFloatVal(-1.1), + cty.NumberIntVal(-1), + cty.NumberIntVal(0), + cty.NumberUIntVal(0), + cty.NumberFloatVal(0), + cty.NumberIntVal(1), + cty.NumberFloatVal(1.1), + cty.NumberUIntVal(2), + }), + expect: `[-1.1, -1, 0, 1, 1.1, 2]`, + }, + { + name: "tuple", + input: cty.TupleVal([]cty.Value{ + cty.StringVal("a"), + cty.BoolVal(true), + cty.NumberFloatVal(0.2), + cty.ListVal([]cty.Value{ + cty.StringVal("a"), cty.StringVal("b"), cty.StringVal("c"), + }), + cty.ObjectVal(map[string]cty.Value{ + "foo": cty.StringVal("bar"), + }), + }), + expect: `["a", true, 0.2, ["a", "b", "c"], {"foo" = "bar"}]`, + }, + } + for _, tc := range testCases { + tc := tc // capture range variable + t.Run(tc.name, func(t *testing.T) { + ci.Parallel(t) + out := printDefault(tc.input) + must.Eq(t, tc.expect, out, must.Sprint(tc.input.GoString())) + }) + } + +} diff --git a/sdk/pack/variables/overrides.go b/sdk/pack/variables/overrides.go new file mode 100644 index 00000000..c22837f9 --- /dev/null +++ b/sdk/pack/variables/overrides.go @@ -0,0 +1,26 @@ +package variables + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/nomad-pack/sdk/pack" + "github.com/zclconf/go-cty/cty" +) + +type Override struct { + Name ID + Path pack.ID + Type cty.Type + Value cty.Value + Range hcl.Range +} + +type Overrides map[pack.ID][]*Override + +func (o *Override) Equal(a *Override) bool { + eq := o.Name == a.Name && + o.Path == a.Path && + o.Range == a.Range && + o.Type == a.Type && + o.Value.RawEquals(a.Value) + return eq +} diff --git a/sdk/pack/variables/variables.go b/sdk/pack/variables/variables.go new file mode 100644 index 00000000..31d72a15 --- /dev/null +++ b/sdk/pack/variables/variables.go @@ -0,0 +1,151 @@ +package variables + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad-pack/internal/pkg/errors/packdiags" + "github.com/hashicorp/nomad-pack/sdk/pack" + + "github.com/hashicorp/hcl/v2" + "github.com/mitchellh/go-wordwrap" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +type PackIDKeyedVarMap map[pack.ID][]*Variable + +type ID string + +func (p ID) String() string { return string(p) } + +// Variable encapsulates a single variable as defined within a block according +// to variableFileSchema and variableBlockSchema. +type Variable struct { + + // Name is the variable label. This is used to identify variables being + // overridden and during templating. + Name ID + + // Description is an optional field which provides additional context to + // users identifying what the variable is used for. + Description string + hasDescription bool + + // Default is an optional field which provides a default value to be used + // in the absence of a user-provided value. It is only in this struct for + // documentation purposes + Default cty.Value + hasDefault bool + + // Type represents the concrete cty type of this variable. If the type is + // unable to be parsed into a cty type, it is invalid. + Type cty.Type + hasType bool + + // Value stores the variable value and is used when converting the cty type + // value into a Go type value. + Value cty.Value + + // DeclRange is the position marker of the variable within the file it was + // read from. This is used for diagnostics. + DeclRange hcl.Range +} + +func (v *Variable) SetDescription(d string) { v.Description = d; v.hasDescription = true } +func (v *Variable) SetDefault(d cty.Value) { v.Default = d; v.hasDefault = true } +func (v *Variable) SetType(t cty.Type) { v.Type = t; v.hasType = true } + +func (v *Variable) Equal(ivp *Variable) bool { + if v == ivp { + return true + } + cv, ov := *v, *ivp + eq := cv.Name == ov.Name && + cv.Description == ov.Description && + cv.hasDescription == ov.hasDescription && + cv.Default == ov.Default && + cv.hasDefault == ov.hasDefault && + cv.Type == ov.Type && + cv.hasType == ov.hasType && + cv.Value == ov.Value + + return eq +} + +func (v *Variable) AsOverrideString(pID pack.ID) string { + var out strings.Builder + out.WriteString(fmt.Sprintf(`# variable "%s.%s"`, pID, v.Name)) + out.WriteByte('\n') + if v.hasDescription { + tmp := "description: " + v.Description + wrapped := wordwrap.WrapString(tmp, 80) + lines := strings.Split(wrapped, "\n") + for i, l := range lines { + lines[i] = "# " + l + } + wrapped = strings.Join(lines, "\n") + out.WriteString(wrapped) + out.WriteString("\n") + } + if v.hasType { + out.WriteString(fmt.Sprintf("# type: %s\n", printType(v.Type))) + } + + if v.hasDefault { + out.WriteString(fmt.Sprintf("# default: %s\n", printDefault(v.Default))) + } + + if v.Value.Equals(v.Default).True() { + out.WriteString(fmt.Sprintf("#\n# %s.%s=%s\n\n", pID, v.Name, printDefault(v.Default))) + } else { + out.WriteString(fmt.Sprintf("#\n%s.%s=%s\n\n", pID, v.Name, printDefault(v.Value))) + } + + out.WriteString("\n") + return out.String() +} + +func (v *Variable) Merge(in *Variable) hcl.Diagnostics { + var diags hcl.Diagnostics + if in.Default != cty.NilVal { + v.hasDefault = in.hasDefault + v.Default = in.Default + } + + if in.Value != cty.NilVal { + v.Value = in.Value + } + + if in.Type != cty.NilType { + v.hasType = in.hasType + v.Type = in.Type + } + + if v.Value != cty.NilVal { + val, err := convert.Convert(v.Value, v.Type) + if err != nil { + switch { + case in.Type != cty.NilType && in.Value == cty.NilVal: + diags = diags.Append(packdiags.DiagInvalidDefaultValue( + fmt.Sprintf("Overriding this variable's type constraint has made its default value invalid: %s.", err), + in.DeclRange.Ptr(), + )) + case in.Type == cty.NilType && in.Value != cty.NilVal: + diags = diags.Append(packdiags.DiagInvalidDefaultValue( + fmt.Sprintf("The overridden default value for this variable is not compatible with the variable's type constraint: %s.", err), + in.DeclRange.Ptr(), + )) + default: + diags = diags.Append(packdiags.DiagInvalidDefaultValue( + fmt.Sprintf("This variable's default value is not compatible with its type constraint: %s.", err), + in.DeclRange.Ptr(), + )) + } + } else { + v.Value = val + } + } + + return diags +} diff --git a/terminal/glint.go b/terminal/glint.go index 6bc22a39..45900fa2 100644 --- a/terminal/glint.go +++ b/terminal/glint.go @@ -367,27 +367,22 @@ func (ui *glintUI) ErrorWithContext(err error, sub string, ctx ...string) { // Add the error string as well as the error type to the output. d.Append(glint.Layout( - glint.Style(glint.Text("\tError: "), glint.Bold()), + glint.Style(glint.Text(" Error: "), glint.Bold()), glint.Text(err.Error()), ).Row()) - d.Append(glint.Layout( - glint.Style(glint.Text("\tType: "), glint.Bold()), - glint.Text(fmt.Sprintf("%T", err)), - ).Row()) - // We only want this section once per error output, so we cannot perform // this within the ctx loop. if len(ctx) > 0 { d.Append(glint.Layout( - glint.Style(glint.Text("\tContext: "), glint.Bold()), + glint.Style(glint.Text(" Context: "), glint.Bold()), ).Row()) } // Iterate the addition context items and append these to the output. for _, additionCTX := range ctx { d.Append(glint.Layout( - glint.Style(glint.Text(fmt.Sprintf("\t - %s", additionCTX))), + glint.Style(glint.Text(fmt.Sprintf(" - %s", additionCTX))), ).Row()) } // Add a new line diff --git a/terminal/ui.go b/terminal/ui.go index f7361f26..7f97265d 100644 --- a/terminal/ui.go +++ b/terminal/ui.go @@ -272,27 +272,22 @@ func ErrorWithContext(err error, sub string, ctx ...string) { // Add the error string as well as the error type to the output. d.Append(glint.Layout( - glint.Style(glint.Text("\tError: "), glint.Bold()), + glint.Style(glint.Text(" Error: "), glint.Bold()), glint.Text(err.Error()), ).Row()) - d.Append(glint.Layout( - glint.Style(glint.Text("\tType: "), glint.Bold()), - glint.Text(fmt.Sprintf("%T", err)), - ).Row()) - // We only want this section once per error output, so we cannot perform // this within the ctx loop. if len(ctx) > 0 { d.Append(glint.Layout( - glint.Style(glint.Text("\tContext: "), glint.Bold()), + glint.Style(glint.Text(" Context: "), glint.Bold()), ).Row()) } // Iterate the addition context items and append these to the output. for _, additionCTX := range ctx { d.Append(glint.Layout( - glint.Style(glint.Text(fmt.Sprintf("\t - %s", additionCTX))), + glint.Style(glint.Text(fmt.Sprintf(" - %s", additionCTX))), ).Row()) }