Make the TestHarness class depper

include/analysis/Grader.h

@@ -4,6 +4,7 @@
 #include "config/Config.h"
 #include "tests/Util.h"
 #include "json.hpp"
+#include "testharness/TestHarness.h"
 
 #include <ostream>
 #include <string>
@@ -14,13 +15,13 @@ using JSON = nlohmann::json;
 
 namespace tester {
 
-class Grader {
+class Grader : public TestHarness {
 public:
   // No default constructor.
   Grader() = delete;
 
   // Construct with output file path.
-  explicit Grader(const Config &cfg);
+  Grader(const Config &cfg) : TestHarness(cfg), cfg(cfg) { findTests(); buildResults(); }
 
   void dump(std::ostream &os) const {
     std::string jsonString = outputJson.dump(2);
@@ -34,10 +35,7 @@ class Grader {
 private:
   // Our config.
   const Config &cfg;
-
-  // Our tests.
-  TestModule tests;
-
+
   // The filtered (must have exe and tests) names of all solutions that will be tested.
   std::vector<std::string> names;
 

include/testharness/TestHarness.h

@@ -2,10 +2,10 @@
 #define TESTER_TEST_HARNESS_H
 
 #include "config/Config.h"
-#include "tests/Util.h"
 #include "testharness/ResultManager.h"
 #include "toolchain/ToolChain.h"
 #include "Colors.h"
+#include "tests/TestParser.h"
 
 #include <string>
 #include <map>
@@ -16,16 +16,20 @@ namespace fs = std::filesystem;
 
 namespace tester {
 
+// Test hierarchy types
+typedef std::vector<std::unique_ptr<TestFile>> SubPackage; 
+typedef std::map<std::string, SubPackage> Package;
+typedef std::map<std::string, Package> TestSet;
+
 // Class that manages finding tests and running them.
 class TestHarness {
 public:
   // No default constructor.
   TestHarness() = delete;
 
   // Construct the Tester with a parsed JSON file.
-  TestHarness(const Config &cfg);
+  TestHarness(const Config &cfg) : cfg(cfg), results() { findTests(); }
 
-  // Run the found tests.
   // Returns true if any tests failed, false otherwise.
   bool runTests();
 
@@ -35,20 +39,34 @@ class TestHarness {
   // Get test summary.
   std::string getTestSummary() const;
 
-private:
-  // Runs the accumulated tests against a specific exe and toolchain.
-  // Returns true if any tests failed, false otherwise.
-  bool runTestsForToolChain(std::string tcId, std::string exeName);
+protected:
+  // where all the tests are held.
+  TestSet testSet; 
 
-private:
+  // A separate subpackage, just for invalid tests.
+  SubPackage invalidTests; 
+
+  // let derived classes find tests.
+  void findTests();  
+
+private: 
   // Our input config.
   const Config &cfg;
-
+  
   // The results of the tests.
   ResultManager results;
 
-  // module where all tests are contained
-  TestModule module;
+private:
+  // test running  
+  bool runTestsForToolChain(std::string tcId, std::string exeName); 
+
+  // test finding and filling methods 
+  void addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file); 
+
+  // utility for filling packages with tests
+  void fillSubpackage(SubPackage& subPackage, const fs::path& subPackPath); 
+  void fillSubpackagesRecursive(Package& package, const fs::path& packPath, const std::string& parentKey);  
+  void setupDebugModule(TestSet& testSet, const fs::path &debugPath);
 };
 
 } // End namespace tester

include/tests/TestFile.h

@@ -44,8 +44,9 @@ class TestFile {
   ParseError getErrorState() const { return errorState; }
   std::string getErrorMessage() const;
   double getElapsedTime() const { return elapsedTime; }
-
-  // setters 
+  bool didError() const { return errorState != ParseError::NoError; }
+
+  //  
   void setTestPath(fs::path path) { testPath = path; }
   void setInsPath(fs::path path) { insPath = path; }
   void setOutPath(fs::path path) { outPath = path; }
@@ -54,8 +55,9 @@ class TestFile {
   void setElapsedTime(double elapsed) { elapsedTime = elapsed; }
 
   // if test has any input and if test uses input file specifically
-  bool usesInputStream, usesInputFile, usesOutStream, usesOutFile;
+  bool usesInputStream{false}, usesInputFile{false}, usesOutStream{false}, usesOutFile{false};
 
+  // TODO: 
   friend class TestParser;
 
 protected:
@@ -66,11 +68,11 @@ class TestFile {
   fs::path testPath, insPath, outPath;
 
   // Test file breaks some convention or was unable to parse directives. 
-  ParseError errorState; 
+  ParseError errorState{ParseError::NoError}; 
   std::string errorMsg;
 
   // elapsed time for final toolchain step
-  double elapsedTime;
+  double elapsedTime{0};
 
 };
 

include/tests/TestParser.h

@@ -13,30 +13,25 @@ namespace tester {
 using PathOrError = std::variant<fs::path, ParseError>;
 
 class TestParser {
+
 public:
-  TestParser(TestFile &testfile) 
-    : testfile(testfile), foundInput(false), foundInputFile(false), 
-      foundCheck(false), foundCheckFile(false), inLineComment(false), 
-      inBlockComment(false), inString(false), insByteCount(0), outByteCount(0)
-  {
-    parseTest();
-  };
+  TestParser(TestFile *testfile) : testfile(testfile) { parse(); }
 
-  int parseTest();
-  void validate();
+  // clear the parsing state for the next test. 
+  void parse(); 
 
 private:
-  // Testfile we parse on
-  TestFile &testfile;
-    
+  // testfile we are parsing
+  TestFile *testfile;
+
   // track state of parse
-  bool foundInput, foundInputFile, foundCheck, foundCheckFile;
+  bool foundInput{false}, foundInputFile{false}, foundCheck{false}, foundCheckFile{false};
 
   // track comment state
-  bool inLineComment, inBlockComment, inString; 
+  bool inLineComment{false}, inBlockComment{false}, inString{false}; 
 
   // current input stream size
-  uint32_t insByteCount, outByteCount;
+  uint32_t insByteCount{0}, outByteCount{0};
 
   // determine if we are in a comment while parsing
   void trackCommentState(std::string &line);
@@ -53,7 +48,6 @@ class TestParser {
   ParseError matchInputFileDirective(std::string &line);
   ParseError matchCheckFileDirective(std::string &line);
   ParseError matchDirectives(std::string &line);
-
 };
 
 }; // namespace tester

include/tests/Util.h

@@ -1,22 +1,22 @@
-#include <map>
-#include <vector>
-#include <string>
-#include <memory>
+// #include <map>
+// #include <vector>
+// #include <string>
+// #include <memory>
 
 #include "config/Config.h"
 #include "TestFile.h"
 #include "TestResult.h"
-#include "toolchain/ToolChain.h"
-#include "Colors.h"
+// #include "toolchain/ToolChain.h"
+// #include "Colors.h"
 
 namespace tester {
 
-typedef std::vector<std::unique_ptr<TestFile>> SubPackage; 
-typedef std::map<std::string, SubPackage> Package;
-typedef std::map<std::string, Package> TestModule;
+// typedef std::vector<std::unique_ptr<TestFile>> SubPackage; 
+// typedef std::map<std::string, SubPackage> Package;
+// typedef std::map<std::string, Package> TestModule;
 
-// test finding interface
-void fillModule(const Config &cfg, TestModule& module);
+// // test finding interface
+// void fillModule(const Config &cfg, TestModule& module);
 
 TestResult runTest(TestFile *test, const ToolChain &toolChain, const Config &cfg);
 

src/analysis/Grader.cpp

@@ -8,11 +8,6 @@ namespace {
 
 namespace tester {
 
-Grader::Grader(const Config &cfg) : cfg(cfg), tests(), outputJson(JSON::object()) {
-  fillModule(cfg, tests);  
-  buildResults();
-}
-
 void Grader::buildResults() {
 
   // outputJson["toolchainCount"] = cfg.getToolChains().size();
@@ -21,7 +16,7 @@ void Grader::buildResults() {
 
   // collect summary of grading
   JSON testSummary = JSON::array(); 
-  for (const auto &testPackage : tests) {
+  for (const auto &testPackage : testSet) {
     // First check if the name exists in the executable lists.
     std::string name = testPackage.first;
     names.push_back(name);
@@ -79,7 +74,7 @@ void Grader::buildResults() {
 
         // Iterate over subpackages and the contained tests from the attacker, tracking pass count.
         size_t passCount = 0, testCount = 0;
-        for (const auto &subpackages : tests[attacker]) {
+        for (const auto &subpackages : testSet[attacker]) {
           for (const std::unique_ptr<TestFile> &test : subpackages.second) {
 
             TestResult result = runTest(test.get(), tc, cfg); 

src/main.cpp

@@ -34,18 +34,6 @@ int main(int argc, char **argv) {
 
     // Run our tests.
     failed = t.runTests();
-
-    // Save or print the summary.
-    // std::string summary = t.getTestSummary();
-    // if (cfg.hasSummaryPath()) {
-    //   std::ofstream sumFile(cfg.getSummaryPath());
-    //   sumFile << summary;
-    //   std::cout << "Summary saved to file: " << cfg.getSummaryPath() << '\n';
-    // }
-    // else {
-    //   std::cout << "Summary:\n" << summary;
-    // }
-
   }
   catch (const std::runtime_error &e) {
     std::cout << "Test harness error: " << e.what() << '\n';