diff --git a/docs/reference/latest/expressions.md b/docs/reference/latest/expressions.md index 0b34c02..ff5c07b 100644 --- a/docs/reference/latest/expressions.md +++ b/docs/reference/latest/expressions.md @@ -79,6 +79,16 @@ The `step` context references the step itself and contains the following: | `step.stdout` | string | standard output of the command, only available after command is run | | `step.stderr` | string | standard error of the command, only available after command is run | +### steps + +The `steps` context is a mapping of preceeding steps, indexed by `id`, in the +current test. `steps` contains all the information from the `step` plus the +following: + +| name | type | description | +| ---- | ---- | ----------- | +| `steps..outputs` | object | resolved `outputs` mappings | + ## Functions and Methods ### Status diff --git a/docs/reference/latest/input.md b/docs/reference/latest/input.md index afc832e..5f0c1c3 100644 --- a/docs/reference/latest/input.md +++ b/docs/reference/latest/input.md @@ -33,9 +33,11 @@ Any type clarifications can be found in the glossary at the bottom. | `env` | map(str, istr) | set environment variables for command and expressions in the step, shadows suite and test `env` | `{}` | | `exec` | istr | location to execute `run` command, either a Docker Compose service name or `:host` | **required** | | `expect` | expr or list(expr) | additional success conditions, evaluated after `run` command, all of which must return a truthy value | `[]` | +| `id` | str | identifier for the step, referenced in `steps` context | | | `if` | expr | execute the step, when result is truthy; otherwise, skip | `success()` | | `index` | int | references the index of the container to execute `run` commmand | `1` | | `name` | istr | a human-readable step name | step index | +| `outputs` | map(str, istr) | available to future steps via `steps` context | `{}` | | `repeat` | map(str, any) | presence indicates a step should be retried if the `run` or any `expect` condition fails | `null` | | `repeat.interval` | str | time to wait between retries in Docker Compose healthcheck format, ex: `1m20s` | `1s` | | `repeat.retries` | int | indicates number of retry attempts; retry indefinitely, if omitted | `null` | diff --git a/docs/reference/latest/results-file.md b/docs/reference/latest/results-file.md index 3fb86a5..5f271b6 100644 --- a/docs/reference/latest/results-file.md +++ b/docs/reference/latest/results-file.md @@ -40,6 +40,7 @@ Where steps contain the following keys: | name | type | description | | ---- | ---- | ----------- | +| `id` | string | the step id | | `name` | string | the step name | | `outcome` | string | the overall outcome of the test, one of `passed`, `failed`, or `skipped` | | `error` | string | an error message, if the step failed for any reason | diff --git a/examples/00-intro.yaml b/examples/00-intro.yaml index 4ac533e..e022fec 100644 --- a/examples/00-intro.yaml +++ b/examples/00-intro.yaml @@ -103,3 +103,40 @@ tests: - exec: node1 run: rm -f repeat-test-file + + step-outputs: + name: Step outputs test + outputs: + TEST_OUTPUT: ${{ steps['step-1'].outputs.A_VALUE }} + steps: + - id: step-1 + exec: node1 + run: | + echo '{ "a": 13 }' + outputs: + A_VALUE: ${{ fromJSON(step.stdout).a }} + + - exec: node1 + run: echo -n '13 is the same as ${{ steps['step-1'].outputs.A_VALUE }}' + expect: + - steps['step-1'].outputs.A_VALUE == '13' + - step.stdout == '13 is the same as 13' + - null == steps['step-1'].outputs.NONEXISTENT_VALUE + - null == steps['nonexistent-step-id'] + + previous-test-outputs: + depends: ["step-outputs"] + name: Previous test outputs test + steps: + - id: capture-stdout + exec: node1 + run: echo -n 'Earlier job output was ${{ tests['step-outputs'].outputs.TEST_OUTPUT }}' + expect: + - tests['step-outputs'].outputs.TEST_OUTPUT == '13' + outputs: + STDOUT: ${{ step.stdout }} + + - exec: node1 + run: /bin/true + expect: + - contains(steps['capture-stdout'].outputs.STDOUT, "Earlier job") diff --git a/examples/01-fails.yaml b/examples/01-fails.yaml index 1527b21..d6e2ff0 100644 --- a/examples/01-fails.yaml +++ b/examples/01-fails.yaml @@ -65,3 +65,20 @@ tests: - exec: node1 run: /bin/true expect: throw("Intentional Failure") + + fail-with-outputs: + name: Failing test that defines outputs + outputs: + FAILED_TEST_OUTPUT: ${{ 1 + 1 }} + steps: + - exec: node1 + run: /bin/false + + maybe-check-empty-failure-outputs: + depends: fail-with-outputs + name: Assert that outputs should be empty from failed tests + steps: + - exec: node1 + run: /bin/true + expect: + - tests['fail-with-outputs'].outputs == {} diff --git a/schemas/input.yaml b/schemas/input.yaml index 46c35d6..8874c94 100644 --- a/schemas/input.yaml +++ b/schemas/input.yaml @@ -4,6 +4,10 @@ $defs: type: object propertyNames: { type: string } additionalProperties: { type: string, expression: InterpolatedText } + outputs: + type: object + propertyNames: { type: string } + additionalProperties: { type: string, expression: InterpolatedText } # suite type: object @@ -23,6 +27,7 @@ properties: properties: env: { "$ref": "#/$defs/env" } name: { type: string, expression: InterpolatedText } + outputs: { "$ref": "#/$defs/outputs" } depends: oneOf: - type: string @@ -45,8 +50,10 @@ properties: - { type: string, expression: Expression } - { type: array, items: { type: string, expression: Expression } } name: { type: string, expression: InterpolatedText } + id: { type: string } if: { type: string, expression: Expression, default: "success()" } index: { type: integer } + outputs: { "$ref": "#/$defs/outputs" } run: oneOf: - { type: string, expression: InterpolatedText } diff --git a/schemas/results-file.yaml b/schemas/results-file.yaml index d032f02..1e6bf56 100644 --- a/schemas/results-file.yaml +++ b/schemas/results-file.yaml @@ -46,6 +46,7 @@ properties: additionalProperties: false required: [ name, outcome, start, stop ] properties: + id: { type: string } name: { type: string } outcome: { "$ref": "#/$defs/outcome" } error: { "$ref": "#/$defs/error" } diff --git a/src/dctest/core.cljs b/src/dctest/core.cljs index 2206195..b8d1464 100644 --- a/src/dctest/core.cljs +++ b/src/dctest/core.cljs @@ -219,6 +219,12 @@ Options: {:delay-ms interval :check-fn #(not (failure? %))}))) +(defn evaluate-step-outputs + "Interpolates step outputs using executed step info (stdout/stderr)." + [context step] + (let [context (assoc context :step step)] + (update step :outputs #(interpolate-any context %)))) + (defn skip-if-necessary "Evaluates the step 'if' expression. Marks step as skipped, if 'if' evaluates to falsy; otherwise, returns step unmodified." @@ -235,6 +241,7 @@ Options: - evaluating 'if' - interpolating keys ('env', 'name', ...) - executing 'run' and 'expect' conditions + - interpolating 'outputs' Short-circuits if skipped ('if' is falsy) or any aspect fails (including any errors during interpolation of keys). On failure, fail the step @@ -253,10 +260,16 @@ Options: (update :exec #(interpolate-any context %)) (update :run #(interpolate-any context %)) (->> (execute-step-retries context)) + (->> (evaluate-step-outputs context)) pass!) + ;; If skipped/failed, outputs will be present but uninterpolated + step (if (passed? step) + step + (assoc step :outputs {})) + stop (js/Date.now) step (assoc step :start start :stop stop)] - (select-keys step [:outcome :name :start :stop :error]))) + (select-keys step [:outcome :id :name :start :stop :error :outputs]))) (defn execute-steps "Runs all steps for a test. Returns test with completed steps. @@ -272,12 +285,24 @@ Options: (assoc-in context [:state :failed] true) context) step (execute-step context step) + context (if (:id step) + (assoc-in context [:steps (:id step)] step) + context) test (update test :steps conj step) test (if (failure? step) (fail! test) test)] (P/recur steps test context))))) +(defn evaluate-test-outputs + "Interpolates test outputs using executed step info (step outputs)." + [context test] + (let [steps-by-id (into {} + (for [s (:steps test) :when (:id s)] + [(:id s) s])) + context (assoc context :steps steps-by-id)] + (update test :outputs #(interpolate-any context %)))) + (defn run-test [context suite test] "Takes an uninterpolated test, and runs it to completion. Returns test with final outcome and any keys that were successfully interpolated. @@ -285,6 +310,7 @@ Options: - interpolating keys ('env', 'name', ...) - running all 'steps' + - interpolating 'outputs' Fail the test if any interpolated key throws an error or any step fails. If any key cannot be interpolated successfully, do not run steps, but do @@ -300,7 +326,15 @@ Options: test (pending-> test (update :name #(interpolate-any context %)) (->> (execute-steps context)) + (->> (evaluate-test-outputs context)) pass!) + ;; If skipped/failed, outputs will be present but uninterpolated + test (if (passed? test) + test + (assoc test :outputs {})) + + ;; Clean up for results-file format + test (assoc test :steps (mapv #(dissoc % :outputs) (:steps test))) stop (js/Date.now) test (assoc test :start start :stop stop) @@ -308,7 +342,7 @@ Options: duration-in-sec (js/Math.floor (/ (- stop start) 1000)) _ (log opts " " (short-outcome test) (:name test) (str "(" duration-in-sec "s)"))] - (select-keys test [:id :name :outcome :start :stop :steps :error]))) + (select-keys test [:id :name :outcome :start :stop :steps :error :outputs]))) (defn filter-tests [graph filter-str] (let [raw-list (if (= "*" filter-str) @@ -350,6 +384,7 @@ Options: suite (P/let [[test & tests] tests test (run-test context suite test) + context (assoc-in context [:tests (:id test)] test) suite (update suite :tests conj test) suite (if (failure? test) (fail! suite) @@ -379,11 +414,16 @@ Options: suite (pending-> suite (update :name #(interpolate-any context %)) (update :tests #(resolve-test-order % (:test-filter opts)))) + _ (log opts) _ (log opts " " (:name suite)) suite (pending-> suite (->> (run-tests context)) - pass!)] + pass!) + + ;; Clean up for results-file format + suite (update suite :tests (fn [tests] + (mapv #(dissoc % :outputs) tests)))] (select-keys suite [:outcome :name :tests]))) diff --git a/src/dctest/expressions.cljs b/src/dctest/expressions.cljs index 27d6940..1a3fe76 100644 --- a/src/dctest/expressions.cljs +++ b/src/dctest/expressions.cljs @@ -14,7 +14,7 @@ (declare read-ast) (def supported-contexts - #{"env" "process" "step"}) + #{"env" "process" "tests" "step" "steps"}) (def stdlib ;; {name {:arity number :fn (fn [context & args] ..)}} diff --git a/test/runexamples b/test/runexamples index 6f4c7f8..f643f1c 100755 --- a/test/runexamples +++ b/test/runexamples @@ -66,9 +66,9 @@ eopt="--environ-file ${repo_root}/examples/03-env-file" up -check 5 0 "${copt} " 00-intro.yaml -check 6 7 "${copt} " 00-intro.yaml 01-fails.yaml -check 5 1 " " 00-intro.yaml 01-fails.yaml +check 7 0 "${copt} " 00-intro.yaml +check 9 8 "${copt} " 00-intro.yaml 01-fails.yaml +check 7 1 " " 00-intro.yaml 01-fails.yaml check 12 0 "${copt} " 02-deps.yaml check 4 0 "${copt} ${eopt}" 03-env.yaml check 2 2 "${copt} " 03-env.yaml