Merge pull request #35 from cmput415/meimar/ci-fix

Meimar/ci fix

.github/workflows/ci.yml

@@ -2,19 +2,13 @@ name: '415 Tester CI'
 run-name: ${{ github.actor }}
 on: [push]
 jobs:
-  build-and-test-user:
+  build-and-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: build and run tester
-        run: tests/run_user_tests.sh
-
-  build-and-test-grader:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: build and run tester
-        run: tests/run_grader_tests.sh
+      - uses: actions/checkout@v3 
+
+      - name: build tester
+        run: tests/build.sh
 
+      - name: run single executable tests  
+        run: tests/single_exe_tests/run.sh

include/Colors.h

@@ -5,9 +5,15 @@
 
 namespace Colors {
 
-    inline const std::string GREEN = "\033[32m";
-    inline const std::string RED = "\033[31m";
-    inline const std::string RESET = "\033[0m";
+    inline const std::string 
+        GREEN = "\033[32m",
+        RED = "\033[31m",
+        YELLOW = "\033[33m",
+        BLUE = "\033[34m",
+        MAGENTA = "\033[35m",
+        CYAN = "\033[36m",
+        WHITE = "\033[37m",
+        RESET = "\033[0m";
 }
 
 #endif

include/tests/TestFile.h

@@ -38,11 +38,11 @@ class TestFile {
   uint64_t id;
 
   // getters
-  fs::path getTestPath() { return testPath; }
-  fs::path getInsPath() { return insPath; }
-  fs::path getOutPath() { return outPath; }
+  fs::path getTestPath() const { return testPath; }
+  fs::path getInsPath() const { return insPath; }
+  fs::path getOutPath() const { return outPath; }
   ParseError getErrorState() const { return errorState; }
-  const std::string &getErrorMessage() const { return errorMsg; }
+  std::string getErrorMessage() const;
 
   // setters 
   void setTestPath(fs::path path) { testPath = path; }
@@ -53,7 +53,7 @@ class TestFile {
   void setErrorMsg (std::string msg) { errorMsg = msg; } 
 
   // if test has any input and if test uses input file specifically
-  bool usesInputStream, usesInputFile, usesOut;
+  bool usesInputStream, usesInputFile, usesOutStream, usesOutFile;
 
   friend class TestParser;
 

src/testharness/TestHarness.cpp

@@ -107,93 +107,90 @@ std::string TestHarness::getTestSummary() const {
 }
 
 bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
-  bool failed = false;
-
-  // Get the toolchain to use.
-  ToolChain toolChain = cfg.getToolChain(tcName);
-
-  // Set the toolchain's exe to be tested.
-  const fs::path &exe = cfg.getExecutablePath(exeName);
-  std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
-  toolChain.setTestedExecutable(exe);
-
-  // If we have a runtime, set that as well.
-  if (cfg.hasRuntime(exeName))
-    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
-  else
-    toolChain.setTestedRuntime("");
+    bool failed = false;
 
-  // Say which toolchain.
-  std::cout << "With toolchain: " << tcName << " -> " <<  toolChain.getBriefDescription() << '\n';
+    // Get the toolchain to use.
+    ToolChain toolChain = cfg.getToolChain(tcName);
 
-  // Stat tracking for toolchain tests.
-  unsigned int toolChainCount = 0, toolChainPasses = 0;
-  // track invalid tests
-  std::vector<fs::path> invalidTests;
+    // Set the toolchain's exe to be tested.
+    const fs::path& exe = cfg.getExecutablePath(exeName);
+    std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
+    toolChain.setTestedExecutable(exe);
 
-  // Iterate over each package.
-  for (const auto& [packageName, package]: module) {
-
-    std::cout << "Entering package: " << packageName << '\n';
-    unsigned int packageCount = 0, packagePasses = 0;
+    // If we have a runtime, set that as well.
+    if (cfg.hasRuntime(exeName))
+        toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
+    else
+        toolChain.setTestedRuntime("");
 
-    // Iterate over each subpackage
-    for (const auto& [subPackageName, subPackage] : package) {
+    // Say which toolchain.
+    std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';
 
-      std::cout << "  Entering subpackage: " << subPackageName << '\n';
-      unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();
+    // Stat tracking for toolchain tests.
+    unsigned int toolChainCount = 0, toolChainPasses = 0;
 
-      // Iterate over each test in the package
-      for (const std::unique_ptr<TestFile>& test : subPackage) {
-
-        if (test->getErrorState() == ParseError::NoError) {
+    // Track invalid tests
+    std::vector<TestFile*> invalidTests;
 
-          TestResult result = runTest(test, toolChain, cfg); 
-          results.addResult(exeName, tcName, subPackageName, result);
-
-          std::cout << "    " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET)) 
-            << " " << test->getTestPath().stem().string() << '\n';
-
-          if (result.pass) {
-            ++packagePasses;
-            ++subPackagePasses;
-          } else {
-            failed = true;
-            if (!cfg.isQuiet() && !result.error)
-              std::cout << '\n' << result.diff << '\n';
-          }
-        } else {
-          std::cout << "    " << test->getTestPath().stem().string() << ": "
-                    << "INVALID" << '\n';
-          --subPackageSize;
-          invalidTests.push_back(test->getTestPath()); 
+    // Iterate over each package.
+    for (const auto& [packageName, package] : module) {
+        std::cout << "Entering package: " << packageName << '\n';
+        unsigned int packageCount = 0, packagePasses = 0;
+
+        // Iterate over each subpackage
+        for (const auto& [subPackageName, subPackage] : package) {
+            std::cout << "  Entering subpackage: " << subPackageName << '\n';
+            unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();
+
+            // Iterate over each test in the package
+            for (const std::unique_ptr<TestFile>& test : subPackage) {
+                if (test->getErrorState() == ParseError::NoError) {
+                    TestResult result = runTest(test, toolChain, cfg);
+                    results.addResult(exeName, tcName, subPackageName, result);
+
+                    std::cout << "    " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET))
+                              << " " << test->getTestPath().stem().string() << '\n';
+
+                    if (result.pass) {
+                        ++packagePasses;
+                        ++subPackagePasses;
+                    } else {
+                        failed = true;
+                        if (!cfg.isQuiet() && !result.error)
+                            std::cout << '\n' << result.diff << '\n';
+                    }
+                } else {
+                    std::cout << "    " << (Colors::YELLOW + "[INVALID]" + Colors::RESET)
+                              << " " << test->getTestPath().stem().string() << '\n';
+                    --subPackageSize;
+                    invalidTests.push_back(test.get());
+                }
+            }
+            std::cout << "  Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n';
+
+            // Track how many tests we run.
+            packageCount += subPackageSize;
         }
-      }
-      std::cout << "  Subpackage passed " << subPackagePasses << " / " << subPackageSize
-                << '\n';
 
-      // Track how many tests we run.
-      packageCount += subPackageSize;
-    }
+        // Update the toolchain stats from the package stats.
+        toolChainPasses += packagePasses;
+        toolChainCount += packageCount;
 
-    // Update the toolchain stats from the package stats.
-    toolChainPasses += packagePasses;
-    toolChainCount += packageCount;
+        std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n';
+    }
 
-    std::cout << " Package passed " << packagePasses<< " / " << packageCount << '\n';
-  }
+    std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
+    std::cout << "Invalid " << invalidTests.size() << " / "
+              << toolChainCount + invalidTests.size() << "\n";
 
-  std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
-  std::cout << "Skipped " << invalidTests.size() << " / " 
-            << toolChainCount + invalidTests.size() << "\n"; 
-
-  for (auto& test: invalidTests) {
-    std::cout << "  Skipped: " << test.stem().string() 
-              << " With error: " << "1" << "\n";
-  }
-  std::cout << "\n";
+    for (const TestFile* test : invalidTests) {
+        std::cout << "  Skipped: " << test->getTestPath().filename().stem() << std::endl
+                  << "  Error: " << Colors::YELLOW << test->getErrorMessage() << Colors::RESET << "\n";
+    }
+    std::cout << "\n";
 
-  return failed;
+    return failed;
 }
 
+
 } // End namespace tester

src/tests/TestFile.cpp

@@ -8,15 +8,15 @@ std::string stripFileExtension(const std::string &str) {
   return str.substr(0, lastIdx);
 }
 
-
 } // anonymous namespace
 
 namespace tester {
 
 uint64_t TestFile::nextId = 0;
 
 TestFile::TestFile(const fs::path& path) 
-  : id(++nextId), usesInputStream(false), usesInputFile(false), usesOut(false), testPath(path),
+  : id(++nextId), usesInputStream(false), usesInputFile(false), 
+    usesOutStream(false), usesOutFile(false), testPath(path),
     errorState(ParseError::NoError)
 {
 
@@ -36,9 +36,39 @@ TestFile::TestFile(const fs::path& path)
 }
 
 TestFile::~TestFile() {
+  // clean up allocated resources on Testfile de-allocation 
   if (usesInputStream && !usesInputFile) {
     fs::remove(insPath);
   }
+  if (usesOutStream && !usesOutFile) {
+    fs::remove(outPath);
+  }
+}
+
+std::string TestFile::getErrorMessage() const {
+
+  switch (getErrorState()) {
+    case ParseError::NoError:
+      return "No error";
+      break;
+    case ParseError::DirectiveConflict:
+      return "Two or more testfile directives supplied that can not coexist in one file.";
+      break;
+    case ParseError::MaxInputBytesExceeded:
+      return "Total bytes used for input stream exceeds maximum";
+      break;
+    case ParseError::MaxOutputBytesExceeded:
+      return "Total bytes used for output stream exceeds maximum";
+      break;
+    case ParseError::FileError:
+      return "A filepath provided in the testfile was unable to be located or opened.";
+      break;
+    case ParseError::RuntimeError:
+      return "An unexpected runtime error occured while parsing the testifle.";
+      break;
+    default:
+      return "No matching Parse State";
+  }
 }
 
 } // namespace tester

src/tests/TestParser.cpp

@@ -231,8 +231,10 @@ int TestParser::parseTest() {
         testfile.usesInputStream = true;
     } if (foundInputFile) {
         testfile.usesInputFile = true;
-    } if (foundCheck | foundCheckFile) {
-        testfile.usesOut = true;
+    } if (foundCheck || foundCheckFile) {
+        testfile.usesOutStream = true;
+    } if (foundCheckFile) {
+        testfile.usesOutFile = true;
     }
 
     // check if input directives have exceeded maximum

src/tests/TestRunning.cpp

@@ -166,7 +166,7 @@ std::pair<bool, std::string> diffFiles(const fs::path& file1, const fs::path& fi
 
   if (!ifs1.is_open() || !ifs2.is_open()) {
     std::cerr << "Error opening files." << std::endl;
-    return std::make_pair(false, "");
+    return std::make_pair(true, "");
   }
 
   std::string line1, line2;

tests/run_user_tests.sh → tests/build.sh

@@ -3,8 +3,6 @@
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 PROJECT_BASE="$SCRIPT_DIR/.."
 
-echo "Project Base: $PROJECT_BASE"
-
 if [ -f $PROJECT_BASE/bin/tester ]; then
   echo "Tester binary found"
 else
@@ -20,12 +18,6 @@ else
   cd -
 fi
 
-# we need to be in the test dir to run tests
-cd $SCRIPT_DIR
-
-# run C tests
-$PROJECT_BASE/bin/tester $SCRIPT_DIR/UserTestConfig.json
-
 status=$?
 
 if [ $status -eq 0 ]; then
@@ -35,3 +27,4 @@ else
 fi
 
 exit $status
+

tests/GradeTestConfig.json → tests/multi_exe_tests/GradeConfig.json

@@ -1,5 +1,5 @@
 {
-  "testDir": "grade_tests/",
+  "testDir": "tests",
   "testedExecutablePaths": {
     "team1": "/usr/bin/clang",
     "team2": "/usr/bin/clang",