From 47e84774f12684ee387a9a79c77f94b74614e6c2 Mon Sep 17 00:00:00 2001 From: Justin Meimar Date: Thu, 27 Jun 2024 10:48:11 -0600 Subject: [PATCH] Fix error test bug when extra stderr supplied --- src/tests/TestRunning.cpp | 71 ++++++++++++------- tests/fail_tests/tests/fail_tests/002_error.c | 11 +-- tests/fail_tests/tests/fail_tests/003_error.c | 8 +++ 3 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 tests/fail_tests/tests/fail_tests/003_error.c diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp index 59abcfce..834762e5 100644 --- a/src/tests/TestRunning.cpp +++ b/src/tests/TestRunning.cpp @@ -5,7 +5,7 @@ #include "dtl/dtl.hpp" #include "toolchain/CommandException.h" #include "toolchain/ExecutionState.h" - +#include #include #include #include @@ -59,7 +59,14 @@ std::vector readFileWithNewlines(const fs::path& filepath) { return lines; } -std::pair getDiffString(const fs::path& file1, const fs::path& file2) { +/** + * @brief a precise character by character diff between two files. + * + * @param genFile file path with generated output of final toolchain step + * @param expFile file path with expected output for the testcase. + * @returns a pair with 1) isDiff boolean and 2) diff string (empty if isDiff is false) + */ +std::pair preciseDiff(const fs::path& file1, const fs::path& file2) { std::vector lines1; std::vector lines2; @@ -94,16 +101,16 @@ std::pair getDiffString(const fs::path& file1, const fs::path * Given a file path, return the substring of the first line that conforms to * the error testcase specification. */ -std::string getErrorString(const fs::path stdOutPath) { +std::optionalgetErrorString(const fs::path outPath) { - std::ifstream ins(stdOutPath); // open input file stream of output file of toolchain + std::ifstream ins(outPath); // open input file stream of output file of toolchain if (!ins.is_open()) { throw std::runtime_error("Failed to open the generated output file of the toolchain."); } std::string firstLine; if (!getline(ins, firstLine)) { - return std::string(""); + return std::nullopt; } if (firstLine.find("Error") != std::string::npos) { @@ -116,7 +123,7 @@ std::string getErrorString(const fs::path stdOutPath) { return firstLine.substr(0, firstLine.find(":")); } - return std::string(""); + return std::nullopt; } void formatFileDump(const fs::path& testPath, const fs::path& expOutPath, @@ -130,6 +137,28 @@ void formatFileDump(const fs::path& testPath, const fs::path& expOutPath, std::cout << "-----------------------" << std::endl; } +/** + * @brief custom diff implementation which corresponds to how we compare an error testcase + * in the spec. Currently, we look for the first line in the generated output and match + * if the expected output is a full substring, up to the first colon. + * + * @param genFile path to the file containing generated output of final toolchain step + * @param expFile path to the file containing expected output for the testcase. + * @returns pair indicating 1) success and 2) diff in case of failure + */ +std::pair errorDiff(const fs::path& genFile, const fs::path& expFile) { + + auto genErrorString = getErrorString(genFile); + auto expErrorString = getErrorString(expFile); + + if (genErrorString && expErrorString && *genErrorString == *expErrorString) { + return {false, ""}; + } + + std::string diffStr = ""; + return std::make_pair(true, std::move(diffStr)); +} + } // end anonymous namespace namespace tester { @@ -146,9 +175,6 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg try { eo = toolChain.build(test); genOutPath = eo.getErrorFile(); - genErrorString = getErrorString(eo.getErrorFile()); - expErrorString = getErrorString(test->getOutPath()); - } catch (const CommandException& ce) { // toolchain throws errors only when allowError is false in the config if (cfg.getVerbosity() > 0) { @@ -157,32 +183,29 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg return TestResult(testPath, false, true, ""); } - bool outputDiff = false; - bool testError = false; - - if (eo.getReturnValue() != 0 && !genErrorString.empty() && !expErrorString.empty()) { - outputDiff = (genErrorString == expErrorString) ? false : true; - testError = true; - } else { - auto diffPair = getDiffString(genOutPath, expOutPath); - outputDiff = diffPair.first; // is there a difference between expected and generated - diffString = diffPair.second; // the diff string - } + bool testDiff = false, testError = false; + std::pair result; + result = preciseDiff(genOutPath, expOutPath); + if (result.first) { + result = errorDiff(genOutPath, expOutPath); + } + testDiff = result.first; + diffString = result.second; // if there is a diff in the output, pick the defined way to display it. int verbosity = cfg.getVerbosity(); if (verbosity == 3) { // highest level of verbosity results in printing the full output even for passing tests. formatFileDump(testPath, expOutPath, genOutPath); - } else if (verbosity == 2 && outputDiff) { + } else if (verbosity == 2 && testDiff) { // level two dump the relevant files formatFileDump(testPath, expOutPath, genOutPath); - } else if (verbosity == 1 && outputDiff) { + } else if (verbosity == 1 && testDiff) { // level one simply print the diff string std::cout << diffString << std::endl; } - - return TestResult(testPath, !outputDiff, testError, ""); + + return TestResult(testPath, !testDiff, testError, ""); } } // End namespace tester \ No newline at end of file diff --git a/tests/fail_tests/tests/fail_tests/002_error.c b/tests/fail_tests/tests/fail_tests/002_error.c index d808abe9..3b7da8ad 100644 --- a/tests/fail_tests/tests/fail_tests/002_error.c +++ b/tests/fail_tests/tests/fail_tests/002_error.c @@ -4,15 +4,10 @@ int main() { #if CUSTOM_COMPILE_TIME_ERROR - printf("CompileTimeError: this is some text after."); - // NOTE: - // Unless a program exits with non-zero status code, it can not - // be considered an error test. - exit(0); - #else - printf("%d", undefined_variable); + printf("CompileTimeError on line 7: this is some text after."); + exit(1); #endif return 0; } -//CHECK:CompileTimeError: \ No newline at end of file +//CHECK:CompileTimeError on line 6: \ No newline at end of file diff --git a/tests/fail_tests/tests/fail_tests/003_error.c b/tests/fail_tests/tests/fail_tests/003_error.c new file mode 100644 index 00000000..7b61f9a6 --- /dev/null +++ b/tests/fail_tests/tests/fail_tests/003_error.c @@ -0,0 +1,8 @@ +#include + +int main() { + + fprintf(stdout, "TypeError on line 5: This is an error!"); + exit(1); + return 0; +}