fix: add command file outputs to task evaluation file tests.

wdl-engine/tests/tasks.rs

@@ -26,6 +26,7 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::path::absolute;
 use std::process::exit;
+use std::sync::LazyLock;
 use std::thread::available_parallelism;
 
 use anyhow::Context;
@@ -41,6 +42,7 @@ use futures::StreamExt;
 use futures::stream;
 use path_clean::clean;
 use pretty_assertions::StrComparison;
+use regex::Regex;
 use tempfile::TempDir;
 use walkdir::WalkDir;
 use wdl_analysis::AnalysisResult;
@@ -57,6 +59,11 @@ use wdl_engine::Inputs;
 use wdl_engine::local::LocalTaskExecutionBackend;
 use wdl_engine::v1::TaskEvaluator;
 
+/// Regex used to replace temporary file names in task command files with
+/// consistent names for test baselines.
+static TEMP_FILENAME_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new("tmp[[:alnum:]]{6}").expect("invalid regex"));
+
 /// Finds tests to run as part of the analysis test suite.
 fn find_tests() -> Vec<PathBuf> {
     // Check for filter arguments consisting of test names
@@ -85,30 +92,32 @@ fn find_tests() -> Vec<PathBuf> {
     tests
 }
 
-/// Normalizes a result.
-fn normalize(root: &Path, s: &str) -> String {
-    // Normalize paths separation characters first
-    let s = s
-        .replace("\\", "/")
-        .replace("//", "/")
-        .replace("\r\n", "\n");
-
+/// Strips paths from the given string.
+fn strip_paths<'a>(root: &Path, s: &'a str) -> Cow<'a, str> {
     // Strip any paths that start with the root directory
     if let Some(root) = root.to_str().map(str::to_string) {
         let mut root = root.replace('\\', "/");
         if !root.ends_with('/') {
             root.push('/');
         }
 
-        s.replace(&root, "")
+        s.replace(&root, "").into()
     } else {
-        s
+        s.into()
     }
 }
 
+/// Normalizes a result.
+fn normalize(s: &str) -> String {
+    // Normalize paths separation characters first
+    s.replace("\\", "/")
+        .replace("//", "/")
+        .replace("\r\n", "\n")
+}
+
 /// Compares a single result.
-fn compare_result(root: &Path, path: &Path, result: &str) -> Result<()> {
-    let result = normalize(root, result);
+fn compare_result(path: &Path, result: &str) -> Result<()> {
+    let result = normalize(result);
     if env::var_os("BLESS").is_some() {
         fs::write(path, &result).with_context(|| {
             format!(
@@ -180,17 +189,14 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
         }
     };
 
+    let test_dir = absolute(test).expect("failed to get absolute directory");
+
     // Make any paths specified in the inputs file relative to the test directory
     let task = result
         .document()
         .task_by_name(&name)
         .ok_or_else(|| anyhow!("document does not contain a task named `{name}`"))?;
-    inputs.join_paths(
-        engine.types_mut(),
-        result.document(),
-        task,
-        &absolute(test).expect("failed to get absolute directory"),
-    );
+    inputs.join_paths(engine.types_mut(), result.document(), task, &test_dir);
 
     let dir = TempDir::new().context("failed to create temporary directory")?;
     let mut evaluator = TaskEvaluator::new(&mut engine);
@@ -199,7 +205,7 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
         .await
     {
         Ok(evaluated) => {
-            compare_evaluation_results(test, dir.path(), &evaluated)?;
+            compare_evaluation_results(&test_dir, dir.path(), &evaluated)?;
 
             match evaluated.into_result() {
                 Ok(outputs) => {
@@ -208,8 +214,8 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
                     let mut serializer = serde_json::Serializer::pretty(&mut buffer);
                     outputs.serialize(engine.types(), &mut serializer)?;
                     let outputs = String::from_utf8(buffer).expect("output should be UTF-8");
-                    let outputs_path = test.join("outputs.json");
-                    compare_result(dir.path(), &outputs_path, &outputs)?;
+                    let outputs = strip_paths(dir.path(), &outputs);
+                    compare_result(&test.join("outputs.json"), &outputs)?;
                 }
                 Err(e) => {
                     let error = match e {
@@ -218,9 +224,8 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
                         }
                         EvaluationError::Other(e) => format!("{e:?}"),
                     };
-
-                    let error_path = test.join("error.txt");
-                    compare_result(dir.path(), &error_path, &error)?;
+                    let error = strip_paths(dir.path(), &error);
+                    compare_result(&test.join("error.txt"), &error)?;
                 }
             }
         }
@@ -231,16 +236,25 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> {
                 }
                 EvaluationError::Other(e) => format!("{e:?}"),
             };
-
-            let error_path = test.join("error.txt");
-            compare_result(dir.path(), &error_path, &error)?;
+            let error = strip_paths(dir.path(), &error);
+            compare_result(&test.join("error.txt"), &error)?;
         }
     }
 
     Ok(())
 }
 
-fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask) -> Result<()> {
+fn compare_evaluation_results(
+    test_dir: &Path,
+    temp_dir: &Path,
+    evaluated: &EvaluatedTask,
+) -> Result<()> {
+    let command = fs::read_to_string(evaluated.command()).with_context(|| {
+        format!(
+            "failed to read task command file `{path}`",
+            path = evaluated.command().display()
+        )
+    })?;
     let stdout =
         fs::read_to_string(evaluated.stdout().as_file().unwrap().as_str()).with_context(|| {
             format!(
@@ -256,15 +270,31 @@ fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask
             )
         })?;
 
-    let stdout_path = test.join("stdout");
-    compare_result(dir, &stdout_path, &stdout)?;
+    // Strip both temp paths and test dir (input file) paths from the outputs
+    let command = strip_paths(temp_dir, &command);
+    let mut command = strip_paths(test_dir, &command);
+
+    // Replace any temporary file names in the command
+    for i in 0..usize::MAX {
+        match TEMP_FILENAME_REGEX.replace(&command, format!("tmp{i}")) {
+            Cow::Borrowed(_) => break,
+            Cow::Owned(s) => command = s.into(),
+        }
+    }
+
+    compare_result(&test_dir.join("command"), &command)?;
+
+    let stdout = strip_paths(temp_dir, &stdout);
+    let stdout = strip_paths(test_dir, &stdout);
+    compare_result(&test_dir.join("stdout"), &stdout)?;
 
-    let stderr_path = test.join("stderr");
-    compare_result(dir, &stderr_path, &stderr)?;
+    let stderr = strip_paths(temp_dir, &stderr);
+    let stderr = strip_paths(test_dir, &stderr);
+    compare_result(&test_dir.join("stderr"), &stderr)?;
 
     // Compare expected output files
     let mut had_files = false;
-    let files_dir = test.join("files");
+    let files_dir = test_dir.join("files");
     for entry in WalkDir::new(evaluated.work_dir()) {
         let entry = entry.with_context(|| {
             format!(
@@ -302,7 +332,7 @@ fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask
                 .expect("should have parent directory"),
         )
         .context("failed to create output file directory")?;
-        compare_result(dir, &expected_path, &contents)?;
+        compare_result(&expected_path, &contents)?;
     }
 
     // Look for missing output files

wdl-engine/tests/tasks/change-extension-task/command

@@ -0,0 +1,2 @@
+printf "data" > 'foo.data'
+printf "index" > 'foo.index'

wdl-engine/tests/tasks/default-option-task/command

@@ -0,0 +1,3 @@
+printf foobar > result1
+printf foobar > result2
+printf foobar > result3

wdl-engine/tests/tasks/expressions-task/command

@@ -0,0 +1 @@
+printf "hello" > hello.txt

wdl-engine/tests/tasks/file-output-task/command

@@ -0,0 +1,2 @@
+printf "hello" > foo.hello
+printf "goodbye" > foo.goodbye

wdl-engine/tests/tasks/file-sizes-task/command

@@ -0,0 +1 @@
+printf "this file is 22 bytes/n" > created_file

wdl-engine/tests/tasks/flags-task/command

@@ -0,0 +1,8 @@
+# If `max_matches` is `None`, the command
+# grep -m  hello greetings.txt
+# would evaluate to
+# 'grep -m <pattern> <infile>', which would be an error.
+
+# Instead, make both the flag and the value conditional on `max_matches`
+# being defined.
+grep  hello 'greetings.txt' | wc -l | sed 's/^ */'

wdl-engine/tests/tasks/glob-task/command

@@ -0,0 +1,3 @@
+for i in {1..3}; do
+    printf ${i} > file_${i}.txt
+done

wdl-engine/tests/tasks/hello/command

@@ -0,0 +1 @@
+grep -E 'hello.*' 'greetings.txt'

wdl-engine/tests/tasks/import-structs/command

@@ -0,0 +1 @@
+printf "The patient makes $35000.000000 per hour/n"

wdl-engine/tests/tasks/input-type-qualifiers/command

@@ -0,0 +1,4 @@
+cat 'tmp/tmp0' >> result
+cat 'tmp/tmp1' >> result
+
+cat 'tmp/tmp2' >> result

wdl-engine/tests/tasks/member-access/command

@@ -0,0 +1 @@
+printf "bar"

wdl-engine/tests/tasks/missing-output-file/command

@@ -0,0 +1 @@
+echo this task forgot to write to foo.txt!

wdl-engine/tests/tasks/optional-output-task/command

@@ -0,0 +1,4 @@
+printf "1" > example1.txt
+if false; then
+  printf "2" > example2.txt
+fi

wdl-engine/tests/tasks/outputs-task/command

@@ -0,0 +1,2 @@
+printf 5 > threshold.txt
+touch a.csv b.csv

wdl-engine/tests/tasks/person-struct-task/command

@@ -0,0 +1,8 @@
+printf "Hello Richard! You have 1 test result(s) available./n"
+
+if true; then
+  if [ "1000000" -gt 1000 ]; then
+    currency="USD"
+    printf "Please transfer $currency 500 to continue"
+  fi
+fi

wdl-engine/tests/tasks/primitive-literals/command

@@ -0,0 +1,2 @@
+mkdir -p testdir
+printf "hello" > testdir/hello.txt

wdl-engine/tests/tasks/private-declaration-task/command

@@ -0,0 +1 @@
+head -3 'tmp/tmp0'

wdl-engine/tests/tasks/relative-and-absolute-task/command

@@ -0,0 +1,2 @@
+mkdir -p my/path/to
+printf "something" > my/path/to/something.txt

wdl-engine/tests/tasks/sum-task/command

@@ -0,0 +1 @@
+printf '1 2 3' | awk '{tot=0; for(i=1;i<=NF;i++) tot+=$i; print tot}'

wdl-engine/tests/tasks/task-fail/command

@@ -0,0 +1,2 @@
+>&2 echo this task is going to fail!
+exit 1

wdl-engine/tests/tasks/task-inputs-task/command

@@ -0,0 +1,6 @@
+for i in 1..1; do
+  printf "hello/n"
+done
+if false; then
+  cat 
+fi

wdl-engine/tests/tasks/task-with-comments/command

@@ -0,0 +1,2 @@
+# This comment WILL be included within the command after it has been parsed
+echo 2

wdl-engine/tests/tasks/test-join-paths/command

@@ -0,0 +1,2 @@
+mkdir 'mydir'
+echo -n "hello" > 'mydir/mydata.txt'

wdl-engine/tests/tasks/test-placeholders-task/command

@@ -0,0 +1,4 @@
+# The `read_lines` function reads the lines from a file into an
+# array. The `sep` function concatenates the lines with a space
+# (" ") delimiter. The resulting string is then printed to stdout.
+printf "hello world hi_world hello friend"

wdl-engine/tests/tasks/test-stderr/command

@@ -0,0 +1 @@
+>&2 printf "hello world"

wdl-engine/tests/tasks/test-stdout/command

@@ -0,0 +1 @@
+printf "hello world"

wdl-engine/tests/tasks/true-false-ternary/command

@@ -0,0 +1,3 @@
+# these two commands have the same result
+printf "hello world" > result1
+printf "hello world" > result2