Improve grader output, interface and script

include/analysis/Grader.h

@@ -21,7 +21,7 @@ class Grader : public TestHarness {
   Grader() = delete;
 
   // Construct with output file path.
-  Grader(const Config& cfg) : TestHarness(cfg), cfg(cfg) {
+  Grader(const Config& cfg) : TestHarness(cfg) {
     findTests();
     buildResults();
   }
@@ -36,12 +36,11 @@ class Grader : public TestHarness {
   void buildResults();
 
 private:
-  // Our config.
-  const Config& cfg;
-
-  // The filtered (must have exe and tests) names of all solutions that will
-  // be tested.
-  std::vector<std::string> names;
+  // The filtered (must have exe and tests) names of all solutions to be tested
+  // std::vector<std::string> defenderNames;
+
+  std::vector<std::string> defendingExes;
+  std::vector<std::string> attackingTestPackages;
 
   // Tester tournament results
   JSON outputJson;

include/config/Config.h

@@ -26,8 +26,6 @@ class Config {
   // Config path getters and checkers.
   const fs::path& getGradePath() const { return gradeFilePath; }
   bool hasGradePath() const { return !gradeFilePath.empty(); }
-  const fs::path& getSummaryPath() const { return summaryFilePath; }
-  bool hasSummaryPath() const { return !summaryFilePath.empty(); }
   const fs::path& getTestDirPath() const { return testDirPath; }
   const fs::path& getDebugPath() const { return debugPath; }
 
@@ -44,7 +42,6 @@ class Config {
   const ToolChain& getToolChain(const std::string& name) const { return toolchains.at(name); }
 
   // Config bool getters.
-  bool isQuiet() const { return quiet; }
   bool isTimed() const { return time; }
   bool isMemoryChecked() const { return memory; }
   int getVerbosity() const { return verbosity; }
@@ -69,8 +66,8 @@ class Config {
   ToolChains toolchains;
 
   // Option flags.
-  bool quiet, debug, time, memory;
-  int verbosity;
+  bool debug, time, memory;
+  int verbosity{0};
 
   // The command timeout.
   int64_t timeout;

include/testharness/TestHarness.h

@@ -40,6 +40,9 @@ class TestHarness {
   std::string getTestSummary() const;
 
 protected:
+  // JSON config 
+  const Config& cfg;
+
   // where all the tests are held.
   TestSet testSet;
 
@@ -51,7 +54,6 @@ class TestHarness {
 
 private:
   // Our input config.
-  const Config& cfg;
 
   // The results of the tests.
   ResultManager results;

src/analysis/Grader.cpp

@@ -1,31 +1,42 @@
 #include "analysis/Grader.h"
-
+#include <algorithm>
 #include <iostream>
 
 // File static namespace.
-namespace {}
+namespace {
+
+bool containsString(const std::vector<std::string>& vec, const std::string& str) {
+  return std::find(vec.begin(), vec.end(), str) != vec.end();
+}
+
+}
 
 namespace tester {
 
 void Grader::buildResults() {
 
-  // outputJson["toolchainCount"] = cfg.getToolChains().size();
   outputJson["title"] = "415 Grades";
   outputJson["results"] = JSON::array();
+
+  // infer the name of each time from the executable string
+  for (const auto& exe : cfg.getExecutables()) {
+    std::cout << "exe: " << exe.first << std::endl;
+    defendingExes.push_back(exe.first);   
+  }
 
-  // collect summary of grading
+  // build a preliminary test summary object
   JSON testSummary = JSON::array();
+  attackingTestPackages = defendingExes; // start with a copy to preserve symmetric ordering
   for (const auto& testPackage : testSet) {
-    // First check if the name exists in the executable lists.
-    std::string name = testPackage.first;
-    names.push_back(name);
-    if (!cfg.hasExecutable(name)) {
-      std::cerr << "Test package (" << name << ") missing executable.\n";
-      continue;
-    }
+
+    std::string packageName = testPackage.first;
+
+    if (!containsString(defendingExes, packageName)) {
+      attackingTestPackages.push_back(packageName);
+    } 
 
     // Create the summary item.
-    JSON summaryItem = {{"team", name}};
+    JSON summaryItem = {{"team", packageName}};
     size_t count = 0;
     for (const auto& subpackage : testPackage.second) {
       count += subpackage.second.size();
@@ -42,11 +53,11 @@ void Grader::buildResults() {
     // Table strings.
     std::string toolChainName = toolChain.first;
     JSON toolChainJson = {{"toolchain", toolChain.first}, {"toolchainResults", JSON::array()}};
+    std::cout << "Toolchain: " << toolChain.first << std::endl;
 
-    // Get the toolchain and start running tests. Run over names twice since
-    // it's nxn.
+    // Get the toolchain and start running tests. Run over names twice since it's nxn.
     ToolChain tc = toolChain.second;
-    for (const std::string& defender : names) {
+    for (const std::string& defender : defendingExes) {
 
       JSON defenseResults = {{"defender", defender}, {"defenderResults", JSON::array()}};
       // Set up the tool chain with the defender's executable.
@@ -58,11 +69,12 @@ void Grader::buildResults() {
         tc.setTestedRuntime("");
 
       // Iterate over attackers.
-      for (const std::string& attacker : names) {
-        std::cout << "==== " << toolChainName << " " << attacker << " (attacker) V.S " << defender
-                  << " (defender)"
-                  << "\n";
-
+      int maxNameLength = 20, arrowStart = 10;
+      for (const std::string& attacker : attackingTestPackages) {
+        std::cout << "  " << std::setw(arrowStart) << std::left << "(" + attacker + ")"
+          << std::setw(maxNameLength - arrowStart) << " --> "
+          << std::setw(10) << "(" + defender + ") ";
+
         JSON attackResults = {{"attacker", attacker}, {"timings", JSON::array()}};
 
         // Iterate over subpackages and the contained tests from the
@@ -73,16 +85,27 @@ void Grader::buildResults() {
 
             TestResult result = runTest(test.get(), tc, cfg);
 
-            if (result.pass) {
-              std::cout << ".";
+            if (result.pass && !result.error) {
+              // a regular test that passes
+              std::cout << Colors::GREEN << "." << Colors::RESET;
+              passCount++;
+            } else if (result.pass && result.error) {
+              // a test failed due to error but in the expected manner
+              std::cout << Colors::GREEN << "x" << Colors::RESET; 
               passCount++;
-            } else if (result.error) {
-              std::cout << "x";
+            } else if (!result.pass && result.error) {
+              // a test failed due to error unexpectedly
+              std::cout << Colors::RED << "x" << Colors::RESET;
+            } else {
+              // a test that produced a different output
+              std::cout << Colors::RED << "." << Colors::RESET;
             }
             std::cout.flush();
             testCount++;
-
-            attackResults["timings"].push_back({test->getTestPath(), test->getElapsedTime()});
+            attackResults["timings"].push_back({
+              test->getTestPath().filename(),
+              test->getElapsedTime()
+            });
           }
         }
         // update the test results

src/config/Config.cpp

@@ -23,11 +23,6 @@ Config::Config(int argc, char** argv) : timeout(2l) {
       ->required()
       ->check(CLI::ExistingFile);
 
-  CLI::Option* summaryOpt = app.add_option("--summary", summaryFilePath,
-                                           "Write the test summary to this file instead of stdout");
-
-  CLI::Option* quietFlag = app.add_flag("-q,--quiet", quiet, "Quiet mode, don't print fail diffs");
-
   CLI::Option* gradeOpt =
       app.add_option("--grade", gradeFilePath, "Perform grading analysis and output to this file");
 
@@ -36,9 +31,7 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   app.add_flag("-t,--time", time, "Include the timings (seconds) of each test in the output.");
   app.add_flag_function(
       "-v", [&](size_t count) { verbosity = static_cast<int>(count); }, "Increase verbosity level");
-
-  gradeOpt->excludes(quietFlag)->excludes(summaryOpt);
-
+
   // Parse our command line options. This has the potential to throw
   // CLI::ParseError, but we want it to continue up the tree.
   try {

src/main.cpp

@@ -8,10 +8,6 @@
 
 int main(int argc, char** argv) {
 
-#if defined(DEBUG)
-  std::cout << "415 Tester running in DEBUG mode..." << std::endl;
-#endif
-
   // Build the config and exit if it fails.
   tester::Config cfg(argc, argv);
   if (!cfg.isInitialised())

tests/multi_exe_tests/GradeConfig.json

@@ -28,6 +28,30 @@
         "usesInStr": true,
         "allowError": true
       }
+    ],
+    "LLVM-opt": [
+      {
+        "stepName": "compile",
+        "executablePath": "$EXE",
+        "arguments": [
+          "$INPUT",
+          "-O3", 
+          "-Wno-everything",
+          "-fno-show-column",
+          "-fno-show-source-location", 
+          "-o", "$OUTPUT"
+        ],
+        "output": "test",
+        "allowError": true 
+      },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "output": "-",
+        "usesInStr": true,
+        "allowError": true
+      }
     ] 
   }
 }

tests/multi_exe_tests/tests/TA/000_ta.c

@@ -0,0 +1 @@
+int main() { return 0; }

tests/multi_exe_tests/tests/TA/001_ta.c

@@ -0,0 +1 @@
+int main() { return 0; }

tests/multi_exe_tests/tests/TA/002_ta.c

@@ -0,0 +1 @@
+int main() { return 0; }

tests/multi_exe_tests/tests/team1/002.c

@@ -1,6 +1,7 @@
 
 // INPUT:abc
 int main() {
-
+    
     return 0;
-}
+}
+

tests/multi_exe_tests/tests/team1/003_timeout.c

@@ -0,0 +1,6 @@
+#include <sys/wait.h>
+
+int main() {
+    sleep(10);
+    return 0;
+}

tests/multi_exe_tests/tests/team1/004_error.c

@@ -0,0 +1,23 @@
+/**
+ * Instead of acutally throwing a runtime C error, whose format does not align with 
+ * how the tester parses error tests, emulate the Gazprea spec.  
+*/
+
+#include <stdio.h>
+
+#define EXIT_DIVIDE_BY_ZERO 2
+
+int main() {
+    int a = 1;
+    int b = 0;
+    if (b != 0) {
+        int c = a / b;
+    } else {
+        fprintf(stderr, "DivideByZeroError: a was about to be divided by 0!");
+        exit(EXIT_DIVIDE_BY_ZERO);
+    }  
+
+    return 0;
+}
+
+//CHECK:DivideByZeroError:

tests/multi_exe_tests/tests/team2/001.c → tests/multi_exe_tests/tests/team2/001_mismatch.c

@@ -2,4 +2,4 @@ int main() {
     return 0;
 }
 
-// CHECK:123 
+//CHECK:123

tests/multi_exe_tests/tests/team3/001.c

@@ -1,4 +1,6 @@
+#include <sys/wait.h>
 int main() {
 
+    sleep(1); // short enough not to timeout
     return 0;
 }

tests/multi_exe_tests/tests/timed_tests/001.c

@@ -0,0 +1,17 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+// INPUT:abc
+
+int main() {
+    char c[3];
+    scanf("%c", &c[0]);
+    scanf("%c", &c[1]);
+    scanf("%c", &c[2]);
+
+    printf("%c%c%c", c[2], c[1], c[0]);
+
+    return 0;
+}
+
+// CHECK:cba

tests/multi_exe_tests/tests/timed_tests/002.c

@@ -0,0 +1,14 @@
+#include <sys/wait.h>
+
+#define ITER 100000000
+
+int main() {
+    int j = 0; 
+    for(int i = 0; i < ITER; i++) {
+        if (j + i -1 < 0xFFFF) {
+            j += i;
+        }
+    } 
+
+    return (j < 0xffffffff);
+}

tests/multi_exe_tests/tests/timed_tests/003_timeout.c

@@ -0,0 +1,12 @@
+#include <sys/wait.h>
+
+int main() {
+    int j = 0; 
+    for(int i = 0; i < 1000000; i++) {
+        if (j + i -1 < 0xFFFF) {
+            j += i;
+        }
+    } 
+
+    return (j < 0xffffffff);
+}