Skip to content

Commit

Permalink
Add all files
Browse files Browse the repository at this point in the history
  • Loading branch information
@JustinMeimar
JustinMeimar committed May 28, 2024
1 parent 541de46 commit 7f0bb9a
Show file tree
Hide file tree
Showing 7 changed files with 214 additions and 91 deletions.
17 changes: 7 additions & 10 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,16 @@ name: '415 Tester CI'
run-name: ${{ github.actor }}
on: [push]
jobs:
build-and-test-user:
build-and-test-1:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- uses: actions/checkout@v3
- name: build and run tester
run: tests/run_user_tests.sh
run: tests/single_exe_tests/run_tests.sh

build-and-test-grader:
build-and-test-2:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: build and run tester
run: tests/run_grader_tests.sh

- uses: actions/checkout@v3
- name: Run Grade Tests
run: tests/mutli_exe_tests/run_tests.sh
12 changes: 9 additions & 3 deletions include/Colors.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,15 @@

namespace Colors {

inline const std::string GREEN = "\033[32m";
inline const std::string RED = "\033[31m";
inline const std::string RESET = "\033[0m";
inline const std::string
GREEN = "\033[32m",
RED = "\033[31m",
YELLOW = "\033[33m",
BLUE = "\033[34m",
MAGENTA = "\033[35m",
CYAN = "\033[36m",
WHITE = "\033[37m",
RESET = "\033[0m";
}

#endif
10 changes: 5 additions & 5 deletions include/tests/TestFile.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ class TestFile {
uint64_t id;

// getters
fs::path getTestPath() { return testPath; }
fs::path getInsPath() { return insPath; }
fs::path getOutPath() { return outPath; }
fs::path getTestPath() const { return testPath; }
fs::path getInsPath() const { return insPath; }
fs::path getOutPath() const { return outPath; }
ParseError getErrorState() const { return errorState; }
const std::string &getErrorMessage() const { return errorMsg; }
std::string getErrorMessage() const;

// setters
void setTestPath(fs::path path) { testPath = path; }
Expand All @@ -53,7 +53,7 @@ class TestFile {
void setErrorMsg (std::string msg) { errorMsg = msg; }

// if test has any input and if test uses input file specifically
bool usesInputStream, usesInputFile, usesOut;
bool usesInputStream, usesInputFile, usesOutStream, usesOutFile;

friend class TestParser;

Expand Down
224 changes: 156 additions & 68 deletions src/testharness/TestHarness.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,94 +106,182 @@ std::string TestHarness::getTestSummary() const {
return allInfo.str();
}

bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
bool failed = false;
// bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
// bool failed = false;

// Get the toolchain to use.
ToolChain toolChain = cfg.getToolChain(tcName);
// // Get the toolchain to use.
// ToolChain toolChain = cfg.getToolChain(tcName);

// Set the toolchain's exe to be tested.
const fs::path &exe = cfg.getExecutablePath(exeName);
std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
toolChain.setTestedExecutable(exe);
// // Set the toolchain's exe to be tested.
// const fs::path &exe = cfg.getExecutablePath(exeName);
// std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
// toolChain.setTestedExecutable(exe);

// If we have a runtime, set that as well.
if (cfg.hasRuntime(exeName))
toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
else
toolChain.setTestedRuntime("");
// // If we have a runtime, set that as well.
// if (cfg.hasRuntime(exeName))
// toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
// else
// toolChain.setTestedRuntime("");

// Say which toolchain.
std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';
// // Say which toolchain.
// std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';

// Stat tracking for toolchain tests.
unsigned int toolChainCount = 0, toolChainPasses = 0;
// track invalid tests
std::vector<fs::path> invalidTests;
// // Stat tracking for toolchain tests.
// unsigned int toolChainCount = 0, toolChainPasses = 0;

// Iterate over each package.
for (const auto& [packageName, package]: module) {
// // track invalid tests
// std::vector<const std::unique_ptr<TestFile>&> invalidTests;

// // Iterate over each package.
// for (const auto& [packageName, package]: module) {

std::cout << "Entering package: " << packageName << '\n';
unsigned int packageCount = 0, packagePasses = 0;
// std::cout << "Entering package: " << packageName << '\n';
// unsigned int packageCount = 0, packagePasses = 0;

// Iterate over each subpackage
for (const auto& [subPackageName, subPackage] : package) {
// // Iterate over each subpackage
// for (const auto& [subPackageName, subPackage] : package) {

std::cout << " Entering subpackage: " << subPackageName << '\n';
unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();
// std::cout << " Entering subpackage: " << subPackageName << '\n';
// unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();

// Iterate over each test in the package
for (const std::unique_ptr<TestFile>& test : subPackage) {
// // Iterate over each test in the package
// for (const std::unique_ptr<TestFile>& test : subPackage) {

if (test->getErrorState() == ParseError::NoError) {
// if (test->getErrorState() == ParseError::NoError) {

TestResult result = runTest(test, toolChain, cfg);
results.addResult(exeName, tcName, subPackageName, result);
// TestResult result = runTest(test, toolChain, cfg);
// results.addResult(exeName, tcName, subPackageName, result);

std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET))
<< " " << test->getTestPath().stem().string() << '\n';
// std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET))
// << " " << test->getTestPath().stem().string() << '\n';

if (result.pass) {
++packagePasses;
++subPackagePasses;
} else {
failed = true;
if (!cfg.isQuiet() && !result.error)
std::cout << '\n' << result.diff << '\n';
}
} else {
std::cout << " " << test->getTestPath().stem().string() << ": "
<< "INVALID" << '\n';
--subPackageSize;
invalidTests.push_back(test->getTestPath());
// if (result.pass) {
// ++packagePasses;
// ++subPackagePasses;
// } else {
// failed = true;
// if (!cfg.isQuiet() && !result.error)
// std::cout << '\n' << result.diff << '\n';
// }
// } else {
// std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET)
// << " " << test->getTestPath().stem().string() << '\n';
// --subPackageSize;
// invalidTests.push_back(test);
// }
// }
// std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize
// << '\n';

// // Track how many tests we run.
// packageCount += subPackageSize;
// }

// // Update the toolchain stats from the package stats.
// toolChainPasses += packagePasses;
// toolChainCount += packageCount;

// std::cout << " Package passed " << packagePasses<< " / " << packageCount << '\n';
// }

// std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
// std::cout << "Invalid " << invalidTests.size() << " / "
// << toolChainCount + invalidTests.size() << "\n";

// for (const std::unique_ptr<TestFile>& test: invalidTests) {
// std::cout << " Skipped: " << test->getTestPath()
// << " With error: " << test->getErrorMessage() << "\n";
// }
// std::cout << "\n";

// return failed;
// }

bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
bool failed = false;

// Get the toolchain to use.
ToolChain toolChain = cfg.getToolChain(tcName);

// Set the toolchain's exe to be tested.
const fs::path& exe = cfg.getExecutablePath(exeName);
std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
toolChain.setTestedExecutable(exe);

// If we have a runtime, set that as well.
if (cfg.hasRuntime(exeName))
toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
else
toolChain.setTestedRuntime("");

// Say which toolchain.
std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';

// Stat tracking for toolchain tests.
unsigned int toolChainCount = 0, toolChainPasses = 0;

// Track invalid tests
std::vector<TestFile*> invalidTests;

// Iterate over each package.
for (const auto& [packageName, package] : module) {
std::cout << "Entering package: " << packageName << '\n';
unsigned int packageCount = 0, packagePasses = 0;

// Iterate over each subpackage
for (const auto& [subPackageName, subPackage] : package) {
std::cout << " Entering subpackage: " << subPackageName << '\n';
unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();

// Iterate over each test in the package
for (const std::unique_ptr<TestFile>& test : subPackage) {
if (test->getErrorState() == ParseError::NoError) {
TestResult result = runTest(test, toolChain, cfg);
results.addResult(exeName, tcName, subPackageName, result);

std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET))
<< " " << test->getTestPath().stem().string() << '\n';

if (result.pass) {
++packagePasses;
++subPackagePasses;
} else {
failed = true;
if (!cfg.isQuiet() && !result.error)
std::cout << '\n' << result.diff << '\n';
}
} else {
std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET)
<< " " << test->getTestPath().stem().string() << '\n';
--subPackageSize;
invalidTests.push_back(test.get());
}
}
std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n';

// Track how many tests we run.
packageCount += subPackageSize;
}
}
std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize
<< '\n';

// Track how many tests we run.
packageCount += subPackageSize;
// Update the toolchain stats from the package stats.
toolChainPasses += packagePasses;
toolChainCount += packageCount;

std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n';
}

// Update the toolchain stats from the package stats.
toolChainPasses += packagePasses;
toolChainCount += packageCount;
std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
std::cout << "Invalid " << invalidTests.size() << " / "
<< toolChainCount + invalidTests.size() << "\n";

std::cout << " Package passed " << packagePasses<< " / " << packageCount << '\n';
}

std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
std::cout << "Skipped " << invalidTests.size() << " / "
<< toolChainCount + invalidTests.size() << "\n";

for (auto& test: invalidTests) {
std::cout << " Skipped: " << test.stem().string()
<< " With error: " << "1" << "\n";
}
std::cout << "\n";
for (const TestFile* test : invalidTests) {
std::cout << " Skipped: " << test->getTestPath().filename().stem() << std::endl
<< " Error: " << Colors::YELLOW << test->getErrorMessage() << Colors::RESET << "\n";
}
std::cout << "\n";

return failed;
return failed;
}


} // End namespace tester
34 changes: 32 additions & 2 deletions src/tests/TestFile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ std::string stripFileExtension(const std::string &str) {
return str.substr(0, lastIdx);
}


} // anonymous namespace

namespace tester {

uint64_t TestFile::nextId = 0;

TestFile::TestFile(const fs::path& path)
: id(++nextId), usesInputStream(false), usesInputFile(false), usesOut(false), testPath(path),
: id(++nextId), usesInputStream(false), usesInputFile(false),
usesOutStream(false), usesOutFile(false), testPath(path),
errorState(ParseError::NoError)
{

Expand All @@ -36,9 +36,39 @@ TestFile::TestFile(const fs::path& path)
}

TestFile::~TestFile() {
// clean up allocated resources on Testfile de-allocation
if (usesInputStream && !usesInputFile) {
fs::remove(insPath);
}
if (usesOutStream && !usesOutFile) {
fs::remove(outPath);
}
}

std::string TestFile::getErrorMessage() const {

switch (getErrorState()) {
case ParseError::NoError:
return "No error";
break;
case ParseError::DirectiveConflict:
return "Two or more testfile directives supplied that can not coexist in one file.";
break;
case ParseError::MaxInputBytesExceeded:
return "Total bytes used for input stream exceeds maximum";
break;
case ParseError::MaxOutputBytesExceeded:
return "Total bytes used for output stream exceeds maximum";
break;
case ParseError::FileError:
return "A filepath provided in the testfile was unable to be located or opened.";
break;
case ParseError::RuntimeError:
return "An unexpected runtime error occured while parsing the testifle.";
break;
default:
return "No matching Parse State";
}
}

} // namespace tester
6 changes: 4 additions & 2 deletions src/tests/TestParser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,10 @@ int TestParser::parseTest() {
testfile.usesInputStream = true;
} if (foundInputFile) {
testfile.usesInputFile = true;
} if (foundCheck | foundCheckFile) {
testfile.usesOut = true;
} if (foundCheck || foundCheckFile) {
testfile.usesOutStream = true;
} if (foundCheckFile) {
testfile.usesOutFile = true;
}

// check if input directives have exceeded maximum
Expand Down
Loading

0 comments on commit 7f0bb9a

Please sign in to comment.