From 2a5c1c8d72b6b2efea640a57415cdc15c027c073 Mon Sep 17 00:00:00 2001 From: Jon Smock Date: Wed, 4 Dec 2024 17:04:20 +0000 Subject: [PATCH] Support 'outputs' in tests and steps Allow users to define 'outputs' at the step and test levels, which is a map (like 'env') that is interpolated last, after the 'run' command is executed and checked by 'expect' conditions. Having 'outputs' facilitates cleaner reuse of computed values and comparison of values across containers without resorting to writing files and/or using ':host'. Besides the addition of the new 'outputs' key, we allow adding 'id' to steps and add two new contexts, 'tests' and 'steps', which represent the previously completed tests and steps respectively. The 'tests' context and the 'depends' key are analogous to 'needs' in GitHub Actions; however, we currently do not check that a user expression that references a test with 'tests' also includes that test in their 'depends' graph. If a user references a non-existent test or non-existent output, they get null (not an exception). Due to our "in place" interpolation of tests and steps, we need to clear uninterpolated 'outputs' in the case the test/step failed or was skipped, so that future tests/steps don't observe uninterpolated strings as 'outputs'. Future refactoring of how tests/steps are interpolated will hopefully make this unnecessary. --- docs/reference/latest/expressions.md | 10 ++++++ docs/reference/latest/input.md | 2 ++ docs/reference/latest/results-file.md | 1 + examples/00-intro.yaml | 38 ++++++++++++++++++++ examples/01-fails.yaml | 17 +++++++++ schemas/input.yaml | 7 ++++ schemas/results-file.yaml | 1 + src/dctest/core.cljs | 50 ++++++++++++++++++++++++--- src/dctest/expressions.cljs | 2 +- test/runexamples | 6 ++-- 10 files changed, 126 insertions(+), 8 deletions(-) diff --git a/docs/reference/latest/expressions.md b/docs/reference/latest/expressions.md index 0b34c02..ff5c07b 100644 --- a/docs/reference/latest/expressions.md +++ b/docs/reference/latest/expressions.md @@ -79,6 +79,16 @@ The `step` context references the step itself and contains the following: | `step.stdout` | string | standard output of the command, only available after command is run | | `step.stderr` | string | standard error of the command, only available after command is run | +### steps + +The `steps` context is a mapping of preceeding steps, indexed by `id`, in the +current test. `steps` contains all the information from the `step` plus the +following: + +| name | type | description | +| ---- | ---- | ----------- | +| `steps..outputs` | object | resolved `outputs` mappings | + ## Functions and Methods ### Status diff --git a/docs/reference/latest/input.md b/docs/reference/latest/input.md index afc832e..5f0c1c3 100644 --- a/docs/reference/latest/input.md +++ b/docs/reference/latest/input.md @@ -33,9 +33,11 @@ Any type clarifications can be found in the glossary at the bottom. | `env` | map(str, istr) | set environment variables for command and expressions in the step, shadows suite and test `env` | `{}` | | `exec` | istr | location to execute `run` command, either a Docker Compose service name or `:host` | **required** | | `expect` | expr or list(expr) | additional success conditions, evaluated after `run` command, all of which must return a truthy value | `[]` | +| `id` | str | identifier for the step, referenced in `steps` context | | | `if` | expr | execute the step, when result is truthy; otherwise, skip | `success()` | | `index` | int | references the index of the container to execute `run` commmand | `1` | | `name` | istr | a human-readable step name | step index | +| `outputs` | map(str, istr) | available to future steps via `steps` context | `{}` | | `repeat` | map(str, any) | presence indicates a step should be retried if the `run` or any `expect` condition fails | `null` | | `repeat.interval` | str | time to wait between retries in Docker Compose healthcheck format, ex: `1m20s` | `1s` | | `repeat.retries` | int | indicates number of retry attempts; retry indefinitely, if omitted | `null` | diff --git a/docs/reference/latest/results-file.md b/docs/reference/latest/results-file.md index 3fb86a5..5f271b6 100644 --- a/docs/reference/latest/results-file.md +++ b/docs/reference/latest/results-file.md @@ -40,6 +40,7 @@ Where steps contain the following keys: | name | type | description | | ---- | ---- | ----------- | +| `id` | string | the step id | | `name` | string | the step name | | `outcome` | string | the overall outcome of the test, one of `passed`, `failed`, or `skipped` | | `error` | string | an error message, if the step failed for any reason | diff --git a/examples/00-intro.yaml b/examples/00-intro.yaml index 4ac533e..c1c78ca 100644 --- a/examples/00-intro.yaml +++ b/examples/00-intro.yaml @@ -103,3 +103,41 @@ tests: - exec: node1 run: rm -f repeat-test-file + + step-outputs: + name: Step outputs test + outputs: + TEST_OUTPUT: ${{ steps['step-1'].outputs.A_VALUE }} + steps: + - id: step-1 + exec: node1 + run: | + echo '{ "a": 13 }' + outputs: + A_VALUE: ${{ fromJSON(step.stdout).a }} + + - exec: node1 + run: echo -n '13 is the same as ${{ steps['step-1'].outputs.A_VALUE }}' + expect: + - steps['step-1'].outputs.A_VALUE == '13' + - step.stdout == '13 is the same as 13' + - null == steps['step-1'].outputs.NONEXISTENT_VALUE + - null == steps['nonexistent-step-id'] + + previous-test-outputs: + # repeat is an arbitrary example of a succesful test without 'outputs' defined + depends: [ repeat, step-outputs ] + name: Check previous tests' outputs + steps: + - name: Successful tests always return truthy 'outputs' + exec: node1 + run: /bin/true + expect: + - tests['repeat'].outputs == {} + + - name: Example step that uses a previous test's outputs + exec: node1 + run: echo -n 'Earlier job output was ${{ tests['step-outputs'].outputs.TEST_OUTPUT }}' + expect: + - step.stdout == 'Earlier job output was 13' + - tests['step-outputs'].outputs.TEST_OUTPUT == '13' diff --git a/examples/01-fails.yaml b/examples/01-fails.yaml index 1527b21..ff1a849 100644 --- a/examples/01-fails.yaml +++ b/examples/01-fails.yaml @@ -65,3 +65,20 @@ tests: - exec: node1 run: /bin/true expect: throw("Intentional Failure") + + fail-with-outputs: + name: Failing test that defines outputs + outputs: + FAILED_TEST_OUTPUT: ${{ 1 + 1 }} + steps: + - exec: node1 + run: /bin/false + + maybe-check-empty-failure-outputs: + depends: fail-with-outputs + name: Assert that outputs should be empty/falsy from failed tests + steps: + - exec: node1 + run: /bin/true + expect: + - tests['fail-with-outputs'].outputs == null diff --git a/schemas/input.yaml b/schemas/input.yaml index 46c35d6..8874c94 100644 --- a/schemas/input.yaml +++ b/schemas/input.yaml @@ -4,6 +4,10 @@ $defs: type: object propertyNames: { type: string } additionalProperties: { type: string, expression: InterpolatedText } + outputs: + type: object + propertyNames: { type: string } + additionalProperties: { type: string, expression: InterpolatedText } # suite type: object @@ -23,6 +27,7 @@ properties: properties: env: { "$ref": "#/$defs/env" } name: { type: string, expression: InterpolatedText } + outputs: { "$ref": "#/$defs/outputs" } depends: oneOf: - type: string @@ -45,8 +50,10 @@ properties: - { type: string, expression: Expression } - { type: array, items: { type: string, expression: Expression } } name: { type: string, expression: InterpolatedText } + id: { type: string } if: { type: string, expression: Expression, default: "success()" } index: { type: integer } + outputs: { "$ref": "#/$defs/outputs" } run: oneOf: - { type: string, expression: InterpolatedText } diff --git a/schemas/results-file.yaml b/schemas/results-file.yaml index d032f02..1e6bf56 100644 --- a/schemas/results-file.yaml +++ b/schemas/results-file.yaml @@ -46,6 +46,7 @@ properties: additionalProperties: false required: [ name, outcome, start, stop ] properties: + id: { type: string } name: { type: string } outcome: { "$ref": "#/$defs/outcome" } error: { "$ref": "#/$defs/error" } diff --git a/src/dctest/core.cljs b/src/dctest/core.cljs index 2206195..668db02 100644 --- a/src/dctest/core.cljs +++ b/src/dctest/core.cljs @@ -219,6 +219,12 @@ Options: {:delay-ms interval :check-fn #(not (failure? %))}))) +(defn evaluate-step-outputs + "Interpolates step outputs using executed step info (stdout/stderr)." + [context step] + (let [context (assoc context :step step)] + (update step :outputs #(interpolate-any context %)))) + (defn skip-if-necessary "Evaluates the step 'if' expression. Marks step as skipped, if 'if' evaluates to falsy; otherwise, returns step unmodified." @@ -235,6 +241,7 @@ Options: - evaluating 'if' - interpolating keys ('env', 'name', ...) - executing 'run' and 'expect' conditions + - interpolating 'outputs' Short-circuits if skipped ('if' is falsy) or any aspect fails (including any errors during interpolation of keys). On failure, fail the step @@ -253,10 +260,16 @@ Options: (update :exec #(interpolate-any context %)) (update :run #(interpolate-any context %)) (->> (execute-step-retries context)) + (->> (evaluate-step-outputs context)) pass!) + ;; If skipped/failed, outputs will be present but uninterpolated + step (if (passed? step) + step + (dissoc step :outputs)) + stop (js/Date.now) step (assoc step :start start :stop stop)] - (select-keys step [:outcome :name :start :stop :error]))) + (select-keys step [:outcome :id :name :start :stop :error :outputs]))) (defn execute-steps "Runs all steps for a test. Returns test with completed steps. @@ -272,12 +285,24 @@ Options: (assoc-in context [:state :failed] true) context) step (execute-step context step) + context (if (:id step) + (assoc-in context [:steps (:id step)] step) + context) test (update test :steps conj step) test (if (failure? step) (fail! test) test)] (P/recur steps test context))))) +(defn evaluate-test-outputs + "Interpolates test outputs using executed step info (step outputs)." + [context test] + (let [steps-by-id (into {} + (for [s (:steps test) :when (:id s)] + [(:id s) s])) + context (assoc context :steps steps-by-id)] + (update test :outputs #(interpolate-any context %)))) + (defn run-test [context suite test] "Takes an uninterpolated test, and runs it to completion. Returns test with final outcome and any keys that were successfully interpolated. @@ -285,6 +310,7 @@ Options: - interpolating keys ('env', 'name', ...) - running all 'steps' + - interpolating 'outputs' Fail the test if any interpolated key throws an error or any step fails. If any key cannot be interpolated successfully, do not run steps, but do @@ -300,7 +326,15 @@ Options: test (pending-> test (update :name #(interpolate-any context %)) (->> (execute-steps context)) + (->> (evaluate-test-outputs context)) pass!) + ;; If skipped/failed, outputs may be present but uninterpolated + test (if (passed? test) + test + (dissoc test :outputs)) + + ;; Clean up for results-file format + test (assoc test :steps (mapv #(dissoc % :outputs) (:steps test))) stop (js/Date.now) test (assoc test :start start :stop stop) @@ -308,7 +342,7 @@ Options: duration-in-sec (js/Math.floor (/ (- stop start) 1000)) _ (log opts " " (short-outcome test) (:name test) (str "(" duration-in-sec "s)"))] - (select-keys test [:id :name :outcome :start :stop :steps :error]))) + (select-keys test [:id :name :outcome :start :stop :steps :error :outputs]))) (defn filter-tests [graph filter-str] (let [raw-list (if (= "*" filter-str) @@ -350,6 +384,7 @@ Options: suite (P/let [[test & tests] tests test (run-test context suite test) + context (assoc-in context [:tests (:id test)] test) suite (update suite :tests conj test) suite (if (failure? test) (fail! suite) @@ -379,11 +414,16 @@ Options: suite (pending-> suite (update :name #(interpolate-any context %)) (update :tests #(resolve-test-order % (:test-filter opts)))) + _ (log opts) _ (log opts " " (:name suite)) suite (pending-> suite (->> (run-tests context)) - pass!)] + pass!) + + ;; Clean up for results-file format + suite (update suite :tests (fn [tests] + (mapv #(dissoc % :outputs) tests)))] (select-keys suite [:outcome :name :tests]))) @@ -502,12 +542,14 @@ Options: (update :expect #(if (string? %) [%] %)) (update :index #(or % 1)) (update :name #(or % (str "steps[" index "]"))) + (update :outputs update-keys name) (update-in [:repeat :interval] parse-interval))) ->test (fn [test id] (-> (merge {:name id} test) (assoc :outcome :pending) (update :env update-keys name) - (update :steps #(vec (map-indexed ->step %))))) + (update :steps #(vec (map-indexed ->step %))) + (update :outputs update-keys name))) ->suite (fn [suite path] (-> (merge {:name path} suite) (assoc :outcome :pending) diff --git a/src/dctest/expressions.cljs b/src/dctest/expressions.cljs index 27d6940..1a3fe76 100644 --- a/src/dctest/expressions.cljs +++ b/src/dctest/expressions.cljs @@ -14,7 +14,7 @@ (declare read-ast) (def supported-contexts - #{"env" "process" "step"}) + #{"env" "process" "tests" "step" "steps"}) (def stdlib ;; {name {:arity number :fn (fn [context & args] ..)}} diff --git a/test/runexamples b/test/runexamples index 6f4c7f8..f643f1c 100755 --- a/test/runexamples +++ b/test/runexamples @@ -66,9 +66,9 @@ eopt="--environ-file ${repo_root}/examples/03-env-file" up -check 5 0 "${copt} " 00-intro.yaml -check 6 7 "${copt} " 00-intro.yaml 01-fails.yaml -check 5 1 " " 00-intro.yaml 01-fails.yaml +check 7 0 "${copt} " 00-intro.yaml +check 9 8 "${copt} " 00-intro.yaml 01-fails.yaml +check 7 1 " " 00-intro.yaml 01-fails.yaml check 12 0 "${copt} " 02-deps.yaml check 4 0 "${copt} ${eopt}" 03-env.yaml check 2 2 "${copt} " 03-env.yaml