From 7f0bb9aebdfff73787415f0e70b9168a60a0b73f Mon Sep 17 00:00:00 2001 From: Justin Meimar Date: Tue, 28 May 2024 16:59:55 -0600 Subject: [PATCH] Add all files --- .github/workflows/ci.yml | 17 +-- include/Colors.h | 12 +- include/tests/TestFile.h | 10 +- src/testharness/TestHarness.cpp | 224 ++++++++++++++++++++++---------- src/tests/TestFile.cpp | 34 ++++- src/tests/TestParser.cpp | 6 +- src/tests/TestRunning.cpp | 2 +- 7 files changed, 214 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b635c2..50cf2237 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,19 +2,16 @@ name: '415 Tester CI' run-name: ${{ github.actor }} on: [push] jobs: - build-and-test-user: + build-and-test-1: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - + - uses: actions/checkout@v3 - name: build and run tester - run: tests/run_user_tests.sh + run: tests/single_exe_tests/run_tests.sh - build-and-test-grader: + build-and-test-2: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: build and run tester - run: tests/run_grader_tests.sh - + - uses: actions/checkout@v3 + - name: Run Grade Tests + run: tests/mutli_exe_tests/run_tests.sh diff --git a/include/Colors.h b/include/Colors.h index d0779eed..3beea4e8 100644 --- a/include/Colors.h +++ b/include/Colors.h @@ -5,9 +5,15 @@ namespace Colors { - inline const std::string GREEN = "\033[32m"; - inline const std::string RED = "\033[31m"; - inline const std::string RESET = "\033[0m"; + inline const std::string + GREEN = "\033[32m", + RED = "\033[31m", + YELLOW = "\033[33m", + BLUE = "\033[34m", + MAGENTA = "\033[35m", + CYAN = "\033[36m", + WHITE = "\033[37m", + RESET = "\033[0m"; } #endif \ No newline at end of file diff --git a/include/tests/TestFile.h b/include/tests/TestFile.h index b31891ab..47ef8961 100644 --- a/include/tests/TestFile.h +++ b/include/tests/TestFile.h @@ -38,11 +38,11 @@ class TestFile { uint64_t id; // getters - fs::path getTestPath() { return testPath; } - fs::path getInsPath() { return insPath; } - fs::path getOutPath() { return outPath; } + fs::path getTestPath() const { return testPath; } + fs::path getInsPath() const { return insPath; } + fs::path getOutPath() const { return outPath; } ParseError getErrorState() const { return errorState; } - const std::string &getErrorMessage() const { return errorMsg; } + std::string getErrorMessage() const; // setters void setTestPath(fs::path path) { testPath = path; } @@ -53,7 +53,7 @@ class TestFile { void setErrorMsg (std::string msg) { errorMsg = msg; } // if test has any input and if test uses input file specifically - bool usesInputStream, usesInputFile, usesOut; + bool usesInputStream, usesInputFile, usesOutStream, usesOutFile; friend class TestParser; diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index 7b70f804..29905bbf 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -106,94 +106,182 @@ std::string TestHarness::getTestSummary() const { return allInfo.str(); } -bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) { - bool failed = false; +// bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) { +// bool failed = false; - // Get the toolchain to use. - ToolChain toolChain = cfg.getToolChain(tcName); +// // Get the toolchain to use. +// ToolChain toolChain = cfg.getToolChain(tcName); - // Set the toolchain's exe to be tested. - const fs::path &exe = cfg.getExecutablePath(exeName); - std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n'; - toolChain.setTestedExecutable(exe); +// // Set the toolchain's exe to be tested. +// const fs::path &exe = cfg.getExecutablePath(exeName); +// std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n'; +// toolChain.setTestedExecutable(exe); - // If we have a runtime, set that as well. - if (cfg.hasRuntime(exeName)) - toolChain.setTestedRuntime(cfg.getRuntimePath(exeName)); - else - toolChain.setTestedRuntime(""); +// // If we have a runtime, set that as well. +// if (cfg.hasRuntime(exeName)) +// toolChain.setTestedRuntime(cfg.getRuntimePath(exeName)); +// else +// toolChain.setTestedRuntime(""); - // Say which toolchain. - std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; +// // Say which toolchain. +// std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; - // Stat tracking for toolchain tests. - unsigned int toolChainCount = 0, toolChainPasses = 0; - // track invalid tests - std::vector invalidTests; +// // Stat tracking for toolchain tests. +// unsigned int toolChainCount = 0, toolChainPasses = 0; - // Iterate over each package. - for (const auto& [packageName, package]: module) { +// // track invalid tests +// std::vector&> invalidTests; + +// // Iterate over each package. +// for (const auto& [packageName, package]: module) { - std::cout << "Entering package: " << packageName << '\n'; - unsigned int packageCount = 0, packagePasses = 0; +// std::cout << "Entering package: " << packageName << '\n'; +// unsigned int packageCount = 0, packagePasses = 0; - // Iterate over each subpackage - for (const auto& [subPackageName, subPackage] : package) { +// // Iterate over each subpackage +// for (const auto& [subPackageName, subPackage] : package) { - std::cout << " Entering subpackage: " << subPackageName << '\n'; - unsigned int subPackagePasses = 0, subPackageSize = subPackage.size(); +// std::cout << " Entering subpackage: " << subPackageName << '\n'; +// unsigned int subPackagePasses = 0, subPackageSize = subPackage.size(); - // Iterate over each test in the package - for (const std::unique_ptr& test : subPackage) { +// // Iterate over each test in the package +// for (const std::unique_ptr& test : subPackage) { - if (test->getErrorState() == ParseError::NoError) { +// if (test->getErrorState() == ParseError::NoError) { - TestResult result = runTest(test, toolChain, cfg); - results.addResult(exeName, tcName, subPackageName, result); +// TestResult result = runTest(test, toolChain, cfg); +// results.addResult(exeName, tcName, subPackageName, result); - std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET)) - << " " << test->getTestPath().stem().string() << '\n'; +// std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET)) +// << " " << test->getTestPath().stem().string() << '\n'; - if (result.pass) { - ++packagePasses; - ++subPackagePasses; - } else { - failed = true; - if (!cfg.isQuiet() && !result.error) - std::cout << '\n' << result.diff << '\n'; - } - } else { - std::cout << " " << test->getTestPath().stem().string() << ": " - << "INVALID" << '\n'; - --subPackageSize; - invalidTests.push_back(test->getTestPath()); +// if (result.pass) { +// ++packagePasses; +// ++subPackagePasses; +// } else { +// failed = true; +// if (!cfg.isQuiet() && !result.error) +// std::cout << '\n' << result.diff << '\n'; +// } +// } else { +// std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) +// << " " << test->getTestPath().stem().string() << '\n'; +// --subPackageSize; +// invalidTests.push_back(test); +// } +// } +// std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize +// << '\n'; + +// // Track how many tests we run. +// packageCount += subPackageSize; +// } + +// // Update the toolchain stats from the package stats. +// toolChainPasses += packagePasses; +// toolChainCount += packageCount; + +// std::cout << " Package passed " << packagePasses<< " / " << packageCount << '\n'; +// } + +// std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n"; +// std::cout << "Invalid " << invalidTests.size() << " / " +// << toolChainCount + invalidTests.size() << "\n"; + +// for (const std::unique_ptr& test: invalidTests) { +// std::cout << " Skipped: " << test->getTestPath() +// << " With error: " << test->getErrorMessage() << "\n"; +// } +// std::cout << "\n"; + +// return failed; +// } + +bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) { + bool failed = false; + + // Get the toolchain to use. + ToolChain toolChain = cfg.getToolChain(tcName); + + // Set the toolchain's exe to be tested. + const fs::path& exe = cfg.getExecutablePath(exeName); + std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n'; + toolChain.setTestedExecutable(exe); + + // If we have a runtime, set that as well. + if (cfg.hasRuntime(exeName)) + toolChain.setTestedRuntime(cfg.getRuntimePath(exeName)); + else + toolChain.setTestedRuntime(""); + + // Say which toolchain. + std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; + + // Stat tracking for toolchain tests. + unsigned int toolChainCount = 0, toolChainPasses = 0; + + // Track invalid tests + std::vector invalidTests; + + // Iterate over each package. + for (const auto& [packageName, package] : module) { + std::cout << "Entering package: " << packageName << '\n'; + unsigned int packageCount = 0, packagePasses = 0; + + // Iterate over each subpackage + for (const auto& [subPackageName, subPackage] : package) { + std::cout << " Entering subpackage: " << subPackageName << '\n'; + unsigned int subPackagePasses = 0, subPackageSize = subPackage.size(); + + // Iterate over each test in the package + for (const std::unique_ptr& test : subPackage) { + if (test->getErrorState() == ParseError::NoError) { + TestResult result = runTest(test, toolChain, cfg); + results.addResult(exeName, tcName, subPackageName, result); + + std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET)) + << " " << test->getTestPath().stem().string() << '\n'; + + if (result.pass) { + ++packagePasses; + ++subPackagePasses; + } else { + failed = true; + if (!cfg.isQuiet() && !result.error) + std::cout << '\n' << result.diff << '\n'; + } + } else { + std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) + << " " << test->getTestPath().stem().string() << '\n'; + --subPackageSize; + invalidTests.push_back(test.get()); + } + } + std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n'; + + // Track how many tests we run. + packageCount += subPackageSize; } - } - std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize - << '\n'; - // Track how many tests we run. - packageCount += subPackageSize; + // Update the toolchain stats from the package stats. + toolChainPasses += packagePasses; + toolChainCount += packageCount; + + std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n'; } - // Update the toolchain stats from the package stats. - toolChainPasses += packagePasses; - toolChainCount += packageCount; + std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n"; + std::cout << "Invalid " << invalidTests.size() << " / " + << toolChainCount + invalidTests.size() << "\n"; - std::cout << " Package passed " << packagePasses<< " / " << packageCount << '\n'; - } - - std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n"; - std::cout << "Skipped " << invalidTests.size() << " / " - << toolChainCount + invalidTests.size() << "\n"; - - for (auto& test: invalidTests) { - std::cout << " Skipped: " << test.stem().string() - << " With error: " << "1" << "\n"; - } - std::cout << "\n"; + for (const TestFile* test : invalidTests) { + std::cout << " Skipped: " << test->getTestPath().filename().stem() << std::endl + << " Error: " << Colors::YELLOW << test->getErrorMessage() << Colors::RESET << "\n"; + } + std::cout << "\n"; - return failed; + return failed; } + } // End namespace tester diff --git a/src/tests/TestFile.cpp b/src/tests/TestFile.cpp index f850a790..db5863a1 100644 --- a/src/tests/TestFile.cpp +++ b/src/tests/TestFile.cpp @@ -8,7 +8,6 @@ std::string stripFileExtension(const std::string &str) { return str.substr(0, lastIdx); } - } // anonymous namespace namespace tester { @@ -16,7 +15,8 @@ namespace tester { uint64_t TestFile::nextId = 0; TestFile::TestFile(const fs::path& path) - : id(++nextId), usesInputStream(false), usesInputFile(false), usesOut(false), testPath(path), + : id(++nextId), usesInputStream(false), usesInputFile(false), + usesOutStream(false), usesOutFile(false), testPath(path), errorState(ParseError::NoError) { @@ -36,9 +36,39 @@ TestFile::TestFile(const fs::path& path) } TestFile::~TestFile() { + // clean up allocated resources on Testfile de-allocation if (usesInputStream && !usesInputFile) { fs::remove(insPath); } + if (usesOutStream && !usesOutFile) { + fs::remove(outPath); + } +} + +std::string TestFile::getErrorMessage() const { + + switch (getErrorState()) { + case ParseError::NoError: + return "No error"; + break; + case ParseError::DirectiveConflict: + return "Two or more testfile directives supplied that can not coexist in one file."; + break; + case ParseError::MaxInputBytesExceeded: + return "Total bytes used for input stream exceeds maximum"; + break; + case ParseError::MaxOutputBytesExceeded: + return "Total bytes used for output stream exceeds maximum"; + break; + case ParseError::FileError: + return "A filepath provided in the testfile was unable to be located or opened."; + break; + case ParseError::RuntimeError: + return "An unexpected runtime error occured while parsing the testifle."; + break; + default: + return "No matching Parse State"; + } } } // namespace tester \ No newline at end of file diff --git a/src/tests/TestParser.cpp b/src/tests/TestParser.cpp index dc2a12ca..c5a107ec 100644 --- a/src/tests/TestParser.cpp +++ b/src/tests/TestParser.cpp @@ -231,8 +231,10 @@ int TestParser::parseTest() { testfile.usesInputStream = true; } if (foundInputFile) { testfile.usesInputFile = true; - } if (foundCheck | foundCheckFile) { - testfile.usesOut = true; + } if (foundCheck || foundCheckFile) { + testfile.usesOutStream = true; + } if (foundCheckFile) { + testfile.usesOutFile = true; } // check if input directives have exceeded maximum diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp index 0f068f9b..4dacd169 100644 --- a/src/tests/TestRunning.cpp +++ b/src/tests/TestRunning.cpp @@ -166,7 +166,7 @@ std::pair diffFiles(const fs::path& file1, const fs::path& fi if (!ifs1.is_open() || !ifs2.is_open()) { std::cerr << "Error opening files." << std::endl; - return std::make_pair(false, ""); + return std::make_pair(true, ""); } std::string line1, line2;