diff --git a/wdl-engine/tests/tasks.rs b/wdl-engine/tests/tasks.rs index 353e83826..d86948ea1 100644 --- a/wdl-engine/tests/tasks.rs +++ b/wdl-engine/tests/tasks.rs @@ -26,6 +26,7 @@ use std::path::Path; use std::path::PathBuf; use std::path::absolute; use std::process::exit; +use std::sync::LazyLock; use std::thread::available_parallelism; use anyhow::Context; @@ -41,6 +42,7 @@ use futures::StreamExt; use futures::stream; use path_clean::clean; use pretty_assertions::StrComparison; +use regex::Regex; use tempfile::TempDir; use walkdir::WalkDir; use wdl_analysis::AnalysisResult; @@ -57,6 +59,11 @@ use wdl_engine::Inputs; use wdl_engine::local::LocalTaskExecutionBackend; use wdl_engine::v1::TaskEvaluator; +/// Regex used to replace temporary file names in task command files with +/// consistent names for test baselines. +static TEMP_FILENAME_REGEX: LazyLock = + LazyLock::new(|| Regex::new("tmp[[:alnum:]]{6}").expect("invalid regex")); + /// Finds tests to run as part of the analysis test suite. fn find_tests() -> Vec { // Check for filter arguments consisting of test names @@ -85,14 +92,8 @@ fn find_tests() -> Vec { tests } -/// Normalizes a result. -fn normalize(root: &Path, s: &str) -> String { - // Normalize paths separation characters first - let s = s - .replace("\\", "/") - .replace("//", "/") - .replace("\r\n", "\n"); - +/// Strips paths from the given string. +fn strip_paths<'a>(root: &Path, s: &'a str) -> Cow<'a, str> { // Strip any paths that start with the root directory if let Some(root) = root.to_str().map(str::to_string) { let mut root = root.replace('\\', "/"); @@ -100,15 +101,23 @@ fn normalize(root: &Path, s: &str) -> String { root.push('/'); } - s.replace(&root, "") + s.replace(&root, "").into() } else { - s + s.into() } } +/// Normalizes a result. +fn normalize(s: &str) -> String { + // Normalize paths separation characters first + s.replace("\\", "/") + .replace("//", "/") + .replace("\r\n", "\n") +} + /// Compares a single result. -fn compare_result(root: &Path, path: &Path, result: &str) -> Result<()> { - let result = normalize(root, result); +fn compare_result(path: &Path, result: &str) -> Result<()> { + let result = normalize(result); if env::var_os("BLESS").is_some() { fs::write(path, &result).with_context(|| { format!( @@ -180,17 +189,14 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> { } }; + let test_dir = absolute(test).expect("failed to get absolute directory"); + // Make any paths specified in the inputs file relative to the test directory let task = result .document() .task_by_name(&name) .ok_or_else(|| anyhow!("document does not contain a task named `{name}`"))?; - inputs.join_paths( - engine.types_mut(), - result.document(), - task, - &absolute(test).expect("failed to get absolute directory"), - ); + inputs.join_paths(engine.types_mut(), result.document(), task, &test_dir); let dir = TempDir::new().context("failed to create temporary directory")?; let mut evaluator = TaskEvaluator::new(&mut engine); @@ -199,7 +205,7 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> { .await { Ok(evaluated) => { - compare_evaluation_results(test, dir.path(), &evaluated)?; + compare_evaluation_results(&test_dir, dir.path(), &evaluated)?; match evaluated.into_result() { Ok(outputs) => { @@ -208,8 +214,8 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> { let mut serializer = serde_json::Serializer::pretty(&mut buffer); outputs.serialize(engine.types(), &mut serializer)?; let outputs = String::from_utf8(buffer).expect("output should be UTF-8"); - let outputs_path = test.join("outputs.json"); - compare_result(dir.path(), &outputs_path, &outputs)?; + let outputs = strip_paths(dir.path(), &outputs); + compare_result(&test.join("outputs.json"), &outputs)?; } Err(e) => { let error = match e { @@ -218,9 +224,8 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> { } EvaluationError::Other(e) => format!("{e:?}"), }; - - let error_path = test.join("error.txt"); - compare_result(dir.path(), &error_path, &error)?; + let error = strip_paths(dir.path(), &error); + compare_result(&test.join("error.txt"), &error)?; } } } @@ -231,16 +236,25 @@ async fn run_test(test: &Path, result: AnalysisResult) -> Result<()> { } EvaluationError::Other(e) => format!("{e:?}"), }; - - let error_path = test.join("error.txt"); - compare_result(dir.path(), &error_path, &error)?; + let error = strip_paths(dir.path(), &error); + compare_result(&test.join("error.txt"), &error)?; } } Ok(()) } -fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask) -> Result<()> { +fn compare_evaluation_results( + test_dir: &Path, + temp_dir: &Path, + evaluated: &EvaluatedTask, +) -> Result<()> { + let command = fs::read_to_string(evaluated.command()).with_context(|| { + format!( + "failed to read task command file `{path}`", + path = evaluated.command().display() + ) + })?; let stdout = fs::read_to_string(evaluated.stdout().as_file().unwrap().as_str()).with_context(|| { format!( @@ -256,15 +270,31 @@ fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask ) })?; - let stdout_path = test.join("stdout"); - compare_result(dir, &stdout_path, &stdout)?; + // Strip both temp paths and test dir (input file) paths from the outputs + let command = strip_paths(temp_dir, &command); + let mut command = strip_paths(test_dir, &command); + + // Replace any temporary file names in the command + for i in 0..usize::MAX { + match TEMP_FILENAME_REGEX.replace(&command, format!("tmp{i}")) { + Cow::Borrowed(_) => break, + Cow::Owned(s) => command = s.into(), + } + } + + compare_result(&test_dir.join("command"), &command)?; + + let stdout = strip_paths(temp_dir, &stdout); + let stdout = strip_paths(test_dir, &stdout); + compare_result(&test_dir.join("stdout"), &stdout)?; - let stderr_path = test.join("stderr"); - compare_result(dir, &stderr_path, &stderr)?; + let stderr = strip_paths(temp_dir, &stderr); + let stderr = strip_paths(test_dir, &stderr); + compare_result(&test_dir.join("stderr"), &stderr)?; // Compare expected output files let mut had_files = false; - let files_dir = test.join("files"); + let files_dir = test_dir.join("files"); for entry in WalkDir::new(evaluated.work_dir()) { let entry = entry.with_context(|| { format!( @@ -302,7 +332,7 @@ fn compare_evaluation_results(test: &Path, dir: &Path, evaluated: &EvaluatedTask .expect("should have parent directory"), ) .context("failed to create output file directory")?; - compare_result(dir, &expected_path, &contents)?; + compare_result(&expected_path, &contents)?; } // Look for missing output files diff --git a/wdl-engine/tests/tasks/array-access/command b/wdl-engine/tests/tasks/array-access/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/array-map-equality/command b/wdl-engine/tests/tasks/array-map-equality/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/change-extension-task/command b/wdl-engine/tests/tasks/change-extension-task/command new file mode 100644 index 000000000..179723fc3 --- /dev/null +++ b/wdl-engine/tests/tasks/change-extension-task/command @@ -0,0 +1,2 @@ +printf "data" > 'foo.data' +printf "index" > 'foo.index' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/compare-coerced/command b/wdl-engine/tests/tasks/compare-coerced/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/compare-optionals/command b/wdl-engine/tests/tasks/compare-optionals/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/concat-optional/command b/wdl-engine/tests/tasks/concat-optional/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/declarations/command b/wdl-engine/tests/tasks/declarations/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/default-option-task/command b/wdl-engine/tests/tasks/default-option-task/command new file mode 100644 index 000000000..5f9f1cf42 --- /dev/null +++ b/wdl-engine/tests/tasks/default-option-task/command @@ -0,0 +1,3 @@ +printf foobar > result1 +printf foobar > result2 +printf foobar > result3 \ No newline at end of file diff --git a/wdl-engine/tests/tasks/expressions-task/command b/wdl-engine/tests/tasks/expressions-task/command new file mode 100644 index 000000000..cce3679f3 --- /dev/null +++ b/wdl-engine/tests/tasks/expressions-task/command @@ -0,0 +1 @@ +printf "hello" > hello.txt \ No newline at end of file diff --git a/wdl-engine/tests/tasks/file-output-task/command b/wdl-engine/tests/tasks/file-output-task/command new file mode 100644 index 000000000..fb131c709 --- /dev/null +++ b/wdl-engine/tests/tasks/file-output-task/command @@ -0,0 +1,2 @@ +printf "hello" > foo.hello +printf "goodbye" > foo.goodbye \ No newline at end of file diff --git a/wdl-engine/tests/tasks/file-sizes-task/command b/wdl-engine/tests/tasks/file-sizes-task/command new file mode 100644 index 000000000..e7281c31d --- /dev/null +++ b/wdl-engine/tests/tasks/file-sizes-task/command @@ -0,0 +1 @@ +printf "this file is 22 bytes/n" > created_file \ No newline at end of file diff --git a/wdl-engine/tests/tasks/flags-task/command b/wdl-engine/tests/tasks/flags-task/command new file mode 100644 index 000000000..f07ddb3f0 --- /dev/null +++ b/wdl-engine/tests/tasks/flags-task/command @@ -0,0 +1,8 @@ +# If `max_matches` is `None`, the command +# grep -m hello greetings.txt +# would evaluate to +# 'grep -m ', which would be an error. + +# Instead, make both the flag and the value conditional on `max_matches` +# being defined. +grep hello 'greetings.txt' | wc -l | sed 's/^ */' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/glob-task/command b/wdl-engine/tests/tasks/glob-task/command new file mode 100644 index 000000000..61ae9b286 --- /dev/null +++ b/wdl-engine/tests/tasks/glob-task/command @@ -0,0 +1,3 @@ +for i in {1..3}; do + printf ${i} > file_${i}.txt +done \ No newline at end of file diff --git a/wdl-engine/tests/tasks/hello/command b/wdl-engine/tests/tasks/hello/command new file mode 100644 index 000000000..365e69b62 --- /dev/null +++ b/wdl-engine/tests/tasks/hello/command @@ -0,0 +1 @@ +grep -E 'hello.*' 'greetings.txt' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/import-structs/command b/wdl-engine/tests/tasks/import-structs/command new file mode 100644 index 000000000..153eb03ef --- /dev/null +++ b/wdl-engine/tests/tasks/import-structs/command @@ -0,0 +1 @@ +printf "The patient makes $35000.000000 per hour/n" \ No newline at end of file diff --git a/wdl-engine/tests/tasks/input-type-qualifiers/command b/wdl-engine/tests/tasks/input-type-qualifiers/command new file mode 100644 index 000000000..cb9e156a5 --- /dev/null +++ b/wdl-engine/tests/tasks/input-type-qualifiers/command @@ -0,0 +1,4 @@ +cat 'tmp/tmp0' >> result +cat 'tmp/tmp1' >> result + +cat 'tmp/tmp2' >> result \ No newline at end of file diff --git a/wdl-engine/tests/tasks/member-access/command b/wdl-engine/tests/tasks/member-access/command new file mode 100644 index 000000000..3f18e1686 --- /dev/null +++ b/wdl-engine/tests/tasks/member-access/command @@ -0,0 +1 @@ +printf "bar" \ No newline at end of file diff --git a/wdl-engine/tests/tasks/missing-output-file/command b/wdl-engine/tests/tasks/missing-output-file/command new file mode 100644 index 000000000..0f2d59068 --- /dev/null +++ b/wdl-engine/tests/tasks/missing-output-file/command @@ -0,0 +1 @@ +echo this task forgot to write to foo.txt! \ No newline at end of file diff --git a/wdl-engine/tests/tasks/multiline-placeholders/command b/wdl-engine/tests/tasks/multiline-placeholders/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/multiline-strings1/command b/wdl-engine/tests/tasks/multiline-strings1/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/multiline-strings2/command b/wdl-engine/tests/tasks/multiline-strings2/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/multiline-strings3/command b/wdl-engine/tests/tasks/multiline-strings3/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/multiline-strings4/command b/wdl-engine/tests/tasks/multiline-strings4/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/nested-access/command b/wdl-engine/tests/tasks/nested-access/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/nested-placeholders/command b/wdl-engine/tests/tasks/nested-placeholders/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/non-empty-optional/command b/wdl-engine/tests/tasks/non-empty-optional/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/optional-output-task/command b/wdl-engine/tests/tasks/optional-output-task/command new file mode 100644 index 000000000..1fb020254 --- /dev/null +++ b/wdl-engine/tests/tasks/optional-output-task/command @@ -0,0 +1,4 @@ +printf "1" > example1.txt +if false; then + printf "2" > example2.txt +fi \ No newline at end of file diff --git a/wdl-engine/tests/tasks/optionals/command b/wdl-engine/tests/tasks/optionals/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/outputs-task/command b/wdl-engine/tests/tasks/outputs-task/command new file mode 100644 index 000000000..472233663 --- /dev/null +++ b/wdl-engine/tests/tasks/outputs-task/command @@ -0,0 +1,2 @@ +printf 5 > threshold.txt +touch a.csv b.csv \ No newline at end of file diff --git a/wdl-engine/tests/tasks/person-struct-task/command b/wdl-engine/tests/tasks/person-struct-task/command new file mode 100644 index 000000000..15a46afb8 --- /dev/null +++ b/wdl-engine/tests/tasks/person-struct-task/command @@ -0,0 +1,8 @@ +printf "Hello Richard! You have 1 test result(s) available./n" + +if true; then + if [ "1000000" -gt 1000 ]; then + currency="USD" + printf "Please transfer $currency 500 to continue" + fi +fi \ No newline at end of file diff --git a/wdl-engine/tests/tasks/placeholder-coercion/command b/wdl-engine/tests/tasks/placeholder-coercion/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/placeholder-none/command b/wdl-engine/tests/tasks/placeholder-none/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/placeholders/command b/wdl-engine/tests/tasks/placeholders/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/primitive-literals/command b/wdl-engine/tests/tasks/primitive-literals/command new file mode 100644 index 000000000..65baa6319 --- /dev/null +++ b/wdl-engine/tests/tasks/primitive-literals/command @@ -0,0 +1,2 @@ +mkdir -p testdir +printf "hello" > testdir/hello.txt \ No newline at end of file diff --git a/wdl-engine/tests/tasks/primitive-to-string/command b/wdl-engine/tests/tasks/primitive-to-string/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/private-declaration-task/command b/wdl-engine/tests/tasks/private-declaration-task/command new file mode 100644 index 000000000..fdb413f46 --- /dev/null +++ b/wdl-engine/tests/tasks/private-declaration-task/command @@ -0,0 +1 @@ +head -3 'tmp/tmp0' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/relative-and-absolute-task/command b/wdl-engine/tests/tasks/relative-and-absolute-task/command new file mode 100644 index 000000000..37f28e61b --- /dev/null +++ b/wdl-engine/tests/tasks/relative-and-absolute-task/command @@ -0,0 +1,2 @@ +mkdir -p my/path/to +printf "something" > my/path/to/something.txt \ No newline at end of file diff --git a/wdl-engine/tests/tasks/seo-option-to-function/command b/wdl-engine/tests/tasks/seo-option-to-function/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/string-to-file/command b/wdl-engine/tests/tasks/string-to-file/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/struct-to-struct/command b/wdl-engine/tests/tasks/struct-to-struct/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/sum-task/command b/wdl-engine/tests/tasks/sum-task/command new file mode 100644 index 000000000..31b348737 --- /dev/null +++ b/wdl-engine/tests/tasks/sum-task/command @@ -0,0 +1 @@ +printf '1 2 3' | awk '{tot=0; for(i=1;i<=NF;i++) tot+=$i; print tot}' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/task-fail/command b/wdl-engine/tests/tasks/task-fail/command new file mode 100644 index 000000000..e23bee76a --- /dev/null +++ b/wdl-engine/tests/tasks/task-fail/command @@ -0,0 +1,2 @@ +>&2 echo this task is going to fail! +exit 1 \ No newline at end of file diff --git a/wdl-engine/tests/tasks/task-inputs-task/command b/wdl-engine/tests/tasks/task-inputs-task/command new file mode 100644 index 000000000..98936b72b --- /dev/null +++ b/wdl-engine/tests/tasks/task-inputs-task/command @@ -0,0 +1,6 @@ +for i in 1..1; do + printf "hello/n" +done +if false; then + cat +fi \ No newline at end of file diff --git a/wdl-engine/tests/tasks/task-with-comments/command b/wdl-engine/tests/tasks/task-with-comments/command new file mode 100644 index 000000000..d934476e1 --- /dev/null +++ b/wdl-engine/tests/tasks/task-with-comments/command @@ -0,0 +1,2 @@ +# This comment WILL be included within the command after it has been parsed +echo 2 \ No newline at end of file diff --git a/wdl-engine/tests/tasks/ternary/command b/wdl-engine/tests/tasks/ternary/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-basename/command b/wdl-engine/tests/tasks/test-basename/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-ceil/command b/wdl-engine/tests/tasks/test-ceil/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-find/command b/wdl-engine/tests/tasks/test-find/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-floor/command b/wdl-engine/tests/tasks/test-floor/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-join-paths/command b/wdl-engine/tests/tasks/test-join-paths/command new file mode 100644 index 000000000..6d4679189 --- /dev/null +++ b/wdl-engine/tests/tasks/test-join-paths/command @@ -0,0 +1,2 @@ +mkdir 'mydir' +echo -n "hello" > 'mydir/mydata.txt' \ No newline at end of file diff --git a/wdl-engine/tests/tasks/test-map/command b/wdl-engine/tests/tasks/test-map/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-matches/command b/wdl-engine/tests/tasks/test-matches/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-max/command b/wdl-engine/tests/tasks/test-max/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-min/command b/wdl-engine/tests/tasks/test-min/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-object/command b/wdl-engine/tests/tasks/test-object/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-pairs/command b/wdl-engine/tests/tasks/test-pairs/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-placeholders-task/command b/wdl-engine/tests/tasks/test-placeholders-task/command new file mode 100644 index 000000000..3779317df --- /dev/null +++ b/wdl-engine/tests/tasks/test-placeholders-task/command @@ -0,0 +1,4 @@ +# The `read_lines` function reads the lines from a file into an +# array. The `sep` function concatenates the lines with a space +# (" ") delimiter. The resulting string is then printed to stdout. +printf "hello world hi_world hello friend" \ No newline at end of file diff --git a/wdl-engine/tests/tasks/test-round/command b/wdl-engine/tests/tasks/test-round/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-stderr/command b/wdl-engine/tests/tasks/test-stderr/command new file mode 100644 index 000000000..cf6c5cfaa --- /dev/null +++ b/wdl-engine/tests/tasks/test-stderr/command @@ -0,0 +1 @@ +>&2 printf "hello world" \ No newline at end of file diff --git a/wdl-engine/tests/tasks/test-stdout/command b/wdl-engine/tests/tasks/test-stdout/command new file mode 100644 index 000000000..53209c7c7 --- /dev/null +++ b/wdl-engine/tests/tasks/test-stdout/command @@ -0,0 +1 @@ +printf "hello world" \ No newline at end of file diff --git a/wdl-engine/tests/tasks/test-struct/command b/wdl-engine/tests/tasks/test-struct/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/test-sub/command b/wdl-engine/tests/tasks/test-sub/command new file mode 100644 index 000000000..e69de29bb diff --git a/wdl-engine/tests/tasks/true-false-ternary/command b/wdl-engine/tests/tasks/true-false-ternary/command new file mode 100644 index 000000000..5dcc3ddaa --- /dev/null +++ b/wdl-engine/tests/tasks/true-false-ternary/command @@ -0,0 +1,3 @@ +# these two commands have the same result +printf "hello world" > result1 +printf "hello world" > result2 \ No newline at end of file