Fixed up CI and reorgnized tests

tests/GradeTestConfig.json → tests/multi_exe_tests/GradeConfig.json

@@ -1,5 +1,5 @@
 {
-  "testDir": "grade_tests/",
+  "testDir": "tests",
   "testedExecutablePaths": {
     "team1": "/usr/bin/clang",
     "team2": "/usr/bin/clang",

tests/multi_exe_tests/grades.json

@@ -0,0 +1,70 @@
+[
+  {
+    "results": [
+      {
+        "attacker": "team1",
+        "defender": "team1",
+        "passCount": 3,
+        "testCount": 3,
+        "timings": []
+      },
+      {
+        "attacker": "team2",
+        "defender": "team1",
+        "passCount": 2,
+        "testCount": 2,
+        "timings": []
+      },
+      {
+        "attacker": "team3",
+        "defender": "team1",
+        "passCount": 1,
+        "testCount": 1,
+        "timings": []
+      },
+      {
+        "attacker": "team1",
+        "defender": "team2",
+        "passCount": 3,
+        "testCount": 3,
+        "timings": []
+      },
+      {
+        "attacker": "team2",
+        "defender": "team2",
+        "passCount": 2,
+        "testCount": 2,
+        "timings": []
+      },
+      {
+        "attacker": "team3",
+        "defender": "team2",
+        "passCount": 1,
+        "testCount": 1,
+        "timings": []
+      },
+      {
+        "attacker": "team1",
+        "defender": "team3",
+        "passCount": 3,
+        "testCount": 3,
+        "timings": []
+      },
+      {
+        "attacker": "team2",
+        "defender": "team3",
+        "passCount": 2,
+        "testCount": 2,
+        "timings": []
+      },
+      {
+        "attacker": "team3",
+        "defender": "team3",
+        "passCount": 1,
+        "testCount": 1,
+        "timings": []
+      }
+    ],
+    "toolchain": "LLVM"
+  }
+]

tests/run_grader_tests.sh → tests/multi_exe_tests/run_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-PROJECT_BASE="$SCRIPT_DIR/.."
+PROJECT_BASE="$SCRIPT_DIR/../.."
 
 echo "Project Base: $PROJECT_BASE"
 
@@ -23,30 +23,21 @@ fi
 # we need to be in the test dir to run tests
 cd $SCRIPT_DIR
 
-echo "Tests passed."
+$PROJECT_BASE/bin/tester $SCRIPT_DIR/GradeConfig.json --grade $SCRIPT_DIR/grades.json
 
 status=$?
 
 if [ $status -eq 0 ]; then
   echo "Passed tests with status: $status"
+  if [ -e $SCRIPT_DIR/grades.json ]; then
+    echo "Grade JSON successfuly created"
+    exit 0 
+  else
+    echo "Failed to output grade JSON"
+    exit 1
+  fi
 else
   echo "Failed tester with exit status: $status"
 fi
 
-exit $status
-
-# TODO: Determine how to assess the output of a grading run
-# in CI.
-
-# run C tests
-# $PROJECT_BASE/bin/tester $SCRIPT_DIR/GradeTestConfig.json --grade grades.csv
-
-# status=$?
-
-# if [ $status -eq 0 ]; then
-#   echo "Passed tests with status: $status"
-# else
-#   echo "Failed tester with exit status: $status"
-# fi
-
-# exit $status
+exit 1

tests/UserTestConfig.json → tests/single_exe_tests/BasicConfig.json

@@ -1,5 +1,5 @@
 {
-  "testDir": "user_tests/",
+  "testDir": "tests",
   "testedExecutablePaths": {
     "clang": "/usr/bin/clang"
   },

tests/single_exe_tests/MutliStageConfig.json

@@ -0,0 +1,31 @@
+{
+  "testDir": "tests",
+  "testedExecutablePaths": {
+    "clang": "/usr/bin/clang"
+  },
+  "toolchains": {
+    "LLVM": [
+      {
+        "stepName": "compile",
+        "executablePath": "$EXE",
+        "arguments": [
+          "$INPUT", 
+          "-Wno-everything",
+          "-fno-show-column",
+          "-fno-show-source-location", 
+          "-o", "$OUTPUT"
+        ],
+        "output": "test",
+        "allowError": true 
+      },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "output": "-",
+        "usesInStr": true,
+        "allowError": true
+      }
+    ] 
+  }
+}

tests/single_exe_tests/MutliToolchain.json

@@ -0,0 +1,51 @@
+{
+  "testDir": "tests",
+  "testedExecutablePaths": {
+    "clang": "/usr/bin/clang"
+  },
+  "toolchains": {
+    "LLVM": [
+      {
+        "stepName": "compile",
+        "executablePath": "$EXE",
+        "arguments": [
+          "$INPUT", 
+          "-Wno-everything",
+          "-fno-show-column",
+          "-fno-show-source-location", 
+          "-o", "$OUTPUT"
+        ],
+        "output": "test",
+        "allowError": true 
+      },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "output": "-",
+        "usesInStr": true,
+        "allowError": true
+      }
+    ] 
+  },
+  "GNU": [
+    {
+      "stepName": "compile",
+      "executablePath": "$EXE",
+      "arguments": [
+        "$INPUT",  
+        "-o", "$OUTPUT"
+      ],
+      "output": "test",
+      "allowError": true 
+    },
+      {
+        "stepName": "run",
+        "executablePath": "$INPUT",
+        "arguments": [],
+        "output": "-",
+        "usesInStr": true,
+        "allowError": true
+      }
+  ]
+}

tests/run_user_tests.sh → tests/single_exe_tests/run_tests.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-PROJECT_BASE="$SCRIPT_DIR/.."
+PROJECT_BASE="$SCRIPT_DIR/../.."
+TEST_CONFIG="$SCRIPT_DIR/BasicConfig.json"
 
 echo "Project Base: $PROJECT_BASE"
 
@@ -24,7 +25,7 @@ fi
 cd $SCRIPT_DIR
 
 # run C tests
-$PROJECT_BASE/bin/tester $SCRIPT_DIR/UserTestConfig.json
+$PROJECT_BASE/bin/tester $TEST_CONFIG
 
 status=$?
 

tests/single_exe_tests/tests/error_tests/001_compile_time_types.c

@@ -0,0 +1,27 @@
+#define TYPE_ERROR 0
+
+typedef struct Apple{
+  char a[5];
+} Apple;
+
+typedef struct Banana {
+  char b[6];
+} Banana ;
+
+int main() {
+
+  Apple a = { "apple" }; 
+  Banana b = { "banana" };
+
+#if TYPE_ERROR 
+  a = b;
+#else
+  // We will emulate the type error to match the Error Test Specification
+  printf("TypeError on line 17: Assigning Banana to Apple."); 
+  exit(1);
+#endif
+
+  return 0;
+}
+
+// CHECK:TypeError on line 17

tests/user_tests/errors/002_runtime_math.c → tests/single_exe_tests/tests/error_tests/002_runtime_math.c

@@ -15,8 +15,8 @@ int main() {
     } else {
         fprintf(stderr, "DivideByZeroError: a was about to be divided by 0!");
         exit(EXIT_DIVIDE_BY_ZERO);
-    }
-
+    }  
+    
     return 0;
 }
 

tests/single_exe_tests/tests/error_tests/003_runtime_index.c

@@ -0,0 +1,28 @@
+#define OUT_OF_BOUNDS_ERROR 0
+
+
+// INPUT:9
+int main() {
+
+    char c;
+    scanf("%c", &c);
+
+    int x = (unsigned int)(c) - '0'; 
+    int a[5] = {1, 2, 3, 4, 5};
+
+    if (x < 5 && x >= 0) {
+        printf("%d", a[x]);
+    } else {
+        // We will emulate the IndexError to match "Error Test Spec" since
+        // C will just let us poke around.
+        printf("IndexError: Don't do that! (The text past the first\
+            colon is implementation specific so is not picked up by the tester)"); 
+
+        exit(1);
+    }
+
+
+    return 0;
+}
+
+// CHECK:IndexError

tests/user_tests/invalid/001_input_exceed.c → tests/single_exe_tests/tests/invalid_tests/001_input_exceed.c

@@ -1,4 +1,4 @@
-//INPUT_FILE:input_exceed.ins
+//INPUT_FILE:./in-stream/input_exceed.ins
 
 #include <stdio.h>
 

tests/user_tests/invalid/002_output_exceed.c → tests/single_exe_tests/tests/invalid_tests/002_output_exceed.c

@@ -1,4 +1,4 @@
-//CHECK_FILE:output_exceed.out
+//CHECK_FILE:./out-stream/output_exceed.out
 
 #include <stdio.h>
 

tests/user_tests/io/003_input_file.c → tests/single_exe_tests/tests/valid_tests/003_input_file.c

@@ -1,4 +1,4 @@
-// INPUT_FILE:./003_input_file.ins
+// INPUT_FILE:./in-stream/003_input_file.ins
 
 #include <stdio.h>
 

tests/user_tests/io/008_escape_seq.c → tests/single_exe_tests/tests/valid_tests/008_escape_seq.c

@@ -3,4 +3,4 @@ int main() {
 	printf("\tafter the tab");
 }
 
-//CHECK_FILE:008_escape_seq.out
+//CHECK_FILE:./out-stream/008_escape_seq.out

tests/user_tests/io/009_tab_newline.c → tests/single_exe_tests/tests/valid_tests/009_tab_newline.c

@@ -6,4 +6,4 @@ int main() {
     return 0;
 }
 
-// CHECK_FILE:009_tab_newline.out
+// CHECK_FILE:./out-stream/009_tab_newline.out