Support 'outputs' in tests and steps

Allow users to define 'outputs' at the step and test levels, which is
a map (like 'env') that is interpolated last, after the 'run' command is
executed and checked by 'expect' conditions. Having 'outputs'
facilitates cleaner reuse of computed values and comparison of values
across containers without resorting to writing files and/or using ':host'.

Besides the addition of the new 'outputs' key, we allow adding 'id' to
steps and add two new contexts, 'tests' and 'steps', which represent the
previously completed tests and steps respectively.

The 'tests' context and the 'depends' key are analogous to 'needs' in
GitHub Actions; however, we currently do not check that a user
expression that references a test with 'tests' also includes that test
in their 'depends' graph. If a user references a non-existent test or
non-existent output, they get null (not an exception).

Due to our "in place" interpolation of tests and steps, we need to clear
uninterpolated 'outputs' in the case the test/step failed or was
skipped, so that future tests/steps don't observe uninterpolated strings
as 'outputs'. Future refactoring of how tests/steps are interpolated
will hopefully make this unnecessary.

docs/reference/latest/expressions.md

@@ -79,6 +79,16 @@ The `step` context references the step itself and contains the following:
 | `step.stdout` | string | standard output of the command, only available after command is run |
 | `step.stderr` | string | standard error of the command, only available after command is run |
 
+### steps
+
+The `steps` context is a mapping of preceeding steps, indexed by `id`, in the
+current test. `steps` contains all the information from the `step` plus the
+following:
+
+| name | type | description |
+| ---- | ---- | ----------- |
+| `steps.<step-id>.outputs` | object | resolved `outputs` mappings |
+
 ## Functions and Methods
 
 ### Status

docs/reference/latest/input.md

@@ -33,9 +33,11 @@ Any type clarifications can be found in the glossary at the bottom.
 | `env` | map(str, istr) | set environment variables for command and expressions in the step, shadows suite and test `env` | `{}` |
 | `exec` | istr | location to execute `run` command, either a Docker Compose service name or `:host` | **required** |
 | `expect` | expr or list(expr) | additional success conditions, evaluated after `run` command, all of which must return a truthy value | `[]` |
+| `id` | str | identifier for the step, referenced in `steps` context | |
 | `if` | expr | execute the step, when result is truthy; otherwise, skip | `success()` |
 | `index` | int | references the index of the container to execute `run` commmand | `1` |
 | `name` | istr | a human-readable step name | step index |
+| `outputs` | map(str, istr) | available to future steps via `steps` context  | `{}` |
 | `repeat` | map(str, any) | presence indicates a step should be retried if the `run` or any `expect` condition fails | `null` |
 | `repeat.interval` | str | time to wait between retries in Docker Compose healthcheck format, ex: `1m20s` | `1s` |
 | `repeat.retries` | int | indicates number of retry attempts; retry indefinitely, if omitted | `null` |

docs/reference/latest/results-file.md

@@ -40,6 +40,7 @@ Where steps contain the following keys:
 
 | name | type | description |
 | ---- | ---- | ----------- |
+| `id` | string | the step id |
 | `name` | string | the step name |
 | `outcome` | string | the overall outcome of the test, one of `passed`, `failed`, or `skipped` |
 | `error` | string | an error message, if the step failed for any reason |

examples/00-intro.yaml

@@ -103,3 +103,40 @@ tests:
 
       - exec: node1
         run: rm -f repeat-test-file
+
+  step-outputs:
+    name: Step outputs test
+    outputs:
+      TEST_OUTPUT: ${{ steps['step-1'].outputs.A_VALUE }}
+    steps:
+      - id: step-1
+        exec: node1
+        run: |
+          echo '{ "a": 13 }'
+        outputs:
+          A_VALUE: ${{ fromJSON(step.stdout).a }}
+
+      - exec: node1
+        run: echo -n '13 is the same as ${{ steps['step-1'].outputs.A_VALUE }}'
+        expect:
+          - steps['step-1'].outputs.A_VALUE == '13'
+          - step.stdout == '13 is the same as 13'
+          - null == steps['step-1'].outputs.NONEXISTENT_VALUE
+          - null == steps['nonexistent-step-id']
+
+  previous-test-outputs:
+    depends: ["step-outputs"]
+    name: Previous test outputs test
+    steps:
+      - id: capture-stdout
+        exec: node1
+        run: echo -n 'Earlier job output was ${{ tests['step-outputs'].outputs.TEST_OUTPUT }}'
+        expect:
+          - tests['step-outputs'].outputs.TEST_OUTPUT == '13'
+        outputs:
+          STDOUT: ${{ step.stdout }}
+
+      - exec: node1
+        run: /bin/true
+        expect:
+          - contains(steps['capture-stdout'].outputs.STDOUT, "Earlier job")

examples/01-fails.yaml

@@ -65,3 +65,20 @@ tests:
       - exec: node1
         run: /bin/true
         expect: throw("Intentional Failure")
+
+  fail-with-outputs:
+    name: Failing test that defines outputs
+    outputs:
+      FAILED_TEST_OUTPUT: ${{ 1 + 1 }}
+    steps:
+      - exec: node1
+        run: /bin/false
+
+  maybe-check-empty-failure-outputs:
+    depends: fail-with-outputs
+    name: Assert that outputs should be empty from failed tests
+    steps:
+      - exec: node1
+        run: /bin/true
+        expect:
+          - tests['fail-with-outputs'].outputs == {}

schemas/input.yaml

@@ -4,6 +4,10 @@ $defs:
     type: object
     propertyNames: { type: string }
     additionalProperties: { type: string, expression: InterpolatedText }
+  outputs:
+    type: object
+    propertyNames: { type: string }
+    additionalProperties: { type: string, expression: InterpolatedText }
 
 # suite
 type: object
@@ -23,6 +27,7 @@ properties:
       properties:
         env:  { "$ref": "#/$defs/env" }
         name: { type: string, expression: InterpolatedText }
+        outputs: { "$ref": "#/$defs/outputs" }
         depends:
           oneOf:
             - type: string
@@ -45,8 +50,10 @@ properties:
                   - { type: string, expression: Expression }
                   - { type: array, items: { type: string, expression: Expression } }
               name: { type: string, expression: InterpolatedText }
+              id: { type: string }
               if: { type: string, expression: Expression, default: "success()" }
               index: { type: integer }
+              outputs: { "$ref": "#/$defs/outputs" }
               run:
                 oneOf:
                   - { type: string, expression: InterpolatedText }

schemas/results-file.yaml

@@ -46,6 +46,7 @@ properties:
             additionalProperties: false
             required: [ name, outcome, start, stop ]
             properties:
+              id: { type: string }
               name: { type: string }
               outcome: { "$ref": "#/$defs/outcome" }
               error: { "$ref": "#/$defs/error" }

src/dctest/core.cljs

@@ -219,6 +219,12 @@ Options:
                        {:delay-ms interval
                         :check-fn #(not (failure? %))})))
 
+(defn evaluate-step-outputs
+  "Interpolates step outputs using executed step info (stdout/stderr)."
+  [context step]
+  (let [context (assoc context :step step)]
+    (update step :outputs #(interpolate-any context %))))
+
 (defn skip-if-necessary
   "Evaluates the step 'if' expression. Marks step as skipped, if 'if'
   evaluates to falsy; otherwise, returns step unmodified."
@@ -235,6 +241,7 @@ Options:
    - evaluating 'if'
    - interpolating keys ('env', 'name', ...)
    - executing 'run' and 'expect' conditions
+   - interpolating 'outputs'
 
   Short-circuits if skipped ('if' is falsy) or any aspect fails (including
   any errors during interpolation of keys). On failure, fail the step
@@ -253,10 +260,16 @@ Options:
                           (update :exec #(interpolate-any context %))
                           (update :run #(interpolate-any context %))
                           (->> (execute-step-retries context))
+                          (->> (evaluate-step-outputs context))
                           pass!)
+          ;; If skipped/failed, outputs will be present but uninterpolated
+          step (if (passed? step)
+                 step
+                 (assoc step :outputs {}))
+
           stop (js/Date.now)
           step (assoc step :start start :stop stop)]
-    (select-keys step [:outcome :name :start :stop :error])))
+    (select-keys step [:outcome :id :name :start :stop :error :outputs])))
 
 (defn execute-steps
   "Runs all steps for a test. Returns test with completed steps.
@@ -272,19 +285,32 @@ Options:
                          (assoc-in context [:state :failed] true)
                          context)
               step (execute-step context step)
+              context (if (:id step)
+                        (assoc-in context [:steps (:id step)] step)
+                        context)
               test (update test :steps conj step)
               test (if (failure? step)
                      (fail! test)
                      test)]
         (P/recur steps test context)))))
 
+(defn evaluate-test-outputs
+  "Interpolates test outputs using executed step info (step outputs)."
+  [context test]
+  (let [steps-by-id (into {}
+                          (for [s (:steps test) :when (:id s)]
+                            [(:id s) s]))
+        context (assoc context :steps steps-by-id)]
+    (update test :outputs #(interpolate-any context %))))
+
 (defn run-test [context suite test]
   "Takes an uninterpolated test, and runs it to completion. Returns test
   with final outcome and any keys that were successfully interpolated.
   Running includes:
 
    - interpolating keys ('env', 'name', ...)
    - running all 'steps'
+   - interpolating 'outputs'
 
   Fail the test if any interpolated key throws an error or any step fails.
   If any key cannot be interpolated successfully, do not run steps, but do
@@ -300,15 +326,23 @@ Options:
           test (pending-> test
                           (update :name #(interpolate-any context %))
                           (->> (execute-steps context))
+                          (->> (evaluate-test-outputs context))
                           pass!)
+          ;; If skipped/failed, outputs will be present but uninterpolated
+          test (if (passed? test)
+                 test
+                 (assoc test :outputs {}))
+
+          ;; Clean up for results-file format
+          test (assoc test :steps (mapv #(dissoc % :outputs) (:steps test)))
 
           stop (js/Date.now)
           test (assoc test :start start :stop stop)
 
           duration-in-sec (js/Math.floor (/ (- stop start) 1000))
           _ (log opts "    " (short-outcome test) (:name test) (str "(" duration-in-sec "s)"))]
 
-    (select-keys test [:id :name :outcome :start :stop :steps :error])))
+    (select-keys test [:id :name :outcome :start :stop :steps :error :outputs])))
 
 (defn filter-tests [graph filter-str]
   (let [raw-list (if (= "*" filter-str)
@@ -350,6 +384,7 @@ Options:
       suite
       (P/let [[test & tests] tests
               test (run-test context suite test)
+              context (assoc-in context [:tests (:id test)] test)
               suite (update suite :tests conj test)
               suite (if (failure? test)
                       (fail! suite)
@@ -379,11 +414,16 @@ Options:
           suite (pending-> suite
                            (update :name #(interpolate-any context %))
                            (update :tests #(resolve-test-order % (:test-filter opts))))
+
           _ (log opts)
           _ (log opts "  " (:name suite))
           suite (pending-> suite
                            (->> (run-tests context))
-                           pass!)]
+                           pass!)
+
+          ;; Clean up for results-file format
+          suite (update suite :tests (fn [tests]
+                                      (mapv #(dissoc % :outputs) tests)))]
 
     (select-keys suite [:outcome :name :tests])))
 

src/dctest/expressions.cljs

@@ -14,7 +14,7 @@
 (declare read-ast)
 
 (def supported-contexts
-  #{"env" "process" "step"})
+  #{"env" "process" "tests" "step" "steps"})
 
 (def stdlib
   ;; {name {:arity number :fn (fn [context & args] ..)}}

test/runexamples

@@ -66,9 +66,9 @@ eopt="--environ-file ${repo_root}/examples/03-env-file"
 
 up
 
-check 5  0 "${copt}        " 00-intro.yaml
-check 6  7 "${copt}        " 00-intro.yaml 01-fails.yaml
-check 5  1 "               " 00-intro.yaml 01-fails.yaml
+check 7  0 "${copt}        " 00-intro.yaml
+check 9  8 "${copt}        " 00-intro.yaml 01-fails.yaml
+check 7  1 "               " 00-intro.yaml 01-fails.yaml
 check 12 0 "${copt}        " 02-deps.yaml
 check 4  0 "${copt} ${eopt}" 03-env.yaml
 check 2  2 "${copt}        " 03-env.yaml