diff --git a/include/toolchain/ToolChain.h b/include/toolchain/ToolChain.h index 4a4fab7c..36365371 100644 --- a/include/toolchain/ToolChain.h +++ b/include/toolchain/ToolChain.h @@ -42,9 +42,6 @@ class ToolChain { // Gets a brief description of the toolchain. std::string getBriefDescription() const; - // Get commands - std::vector& getCommands() { return commands; } - // Ostream operator. friend std::ostream& operator<<(std::ostream&, const ToolChain&); diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index f044d7ce..ed85e66b 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -70,7 +70,6 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; unsigned int toolChainCount = 0, toolChainPasses = 0; // Stat tracking for toolchain tests. - std::vector failedTests; // Iterate over each package. for (auto& [packageName, package] : testSet) { @@ -96,7 +95,6 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) ++subPackagePasses; } else { failed = true; - failedTests.push_back(test.get()); } } else { std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) << " " @@ -124,11 +122,6 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) std::cout << " Skipped: " << test->getTestPath().filename().stem() << std::endl << " Error: " << Colors::YELLOW << test->getParseErrorMsg() << Colors::RESET << "\n"; } - - for (auto test : failedTests) { - std::cout << " Failed: " << test->getTestPath().filename().stem() << std::endl; - } - std::cout << "\n"; return failed; diff --git a/tests/grades.json b/tests/grades.json deleted file mode 100644 index 958371fe..00000000 --- a/tests/grades.json +++ /dev/null @@ -1,925 +0,0 @@ -{ - "results": [ - { - "toolchain": "LLVM", - "toolchainResults": [ - { - "defender": "TA", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010318966 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010668327 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010393714 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010564722 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010429847 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010417658 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010382968 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010340103 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team1", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010549606 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010763091 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010653573 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010863821 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010528791 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010442404 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010418285 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010393463 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team2", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010760151 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010363198 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010312714 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.01032394 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010294179 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010472594 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010481068 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010414413 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team3", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010420623 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.01063799 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010574634 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010566123 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010392695 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010787876 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010435922 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010739763 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - } - ] - }, - { - "toolchain": "LLVM-opt", - "toolchainResults": [ - { - "defender": "TA", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010748125 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010425343 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010651275 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010307127 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010438114 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010693689 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.01062873 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.01049052 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team1", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010442633 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010580033 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010642961 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010672825 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010499728 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010544231 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010708269 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010597546 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team2", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010703711 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.01065987 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.01033922 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010607099 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010756519 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010657796 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010793364 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010419758 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - }, - { - "defender": "team3", - "defenderResults": [ - { - "attacker": "TA", - "passCount": 3, - "testCount": 3, - "timings": [ - { - "pass": true, - "test": "002_ta.c", - "time": 0.010357343 - }, - { - "pass": true, - "test": "001_ta.c", - "time": 0.010610894 - }, - { - "pass": true, - "test": "000_ta.c", - "time": 0.010560141 - } - ] - }, - { - "attacker": "team1", - "passCount": 3, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010459303 - }, - { - "pass": true, - "test": "002.c", - "time": 0.010322641 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - }, - { - "attacker": "team2", - "passCount": 1, - "testCount": 2, - "timings": [ - { - "pass": true, - "test": "002.c", - "time": 0.010608155 - }, - { - "pass": false, - "test": "001_mismatch.c", - "time": 0.010575701 - } - ] - }, - { - "attacker": "team3", - "passCount": 0, - "testCount": 1, - "timings": [ - { - "pass": false, - "test": "001.c", - "time": 0.0 - } - ] - }, - { - "attacker": "timed_tests", - "passCount": 2, - "testCount": 4, - "timings": [ - { - "pass": true, - "test": "004_error.c", - "time": 0.0 - }, - { - "pass": true, - "test": "001.c", - "time": 0.010329693 - }, - { - "pass": false, - "test": "002.c", - "time": 0.0 - }, - { - "pass": false, - "test": "003_timeout.c", - "time": 0.0 - } - ] - } - ] - } - ] - } - ], - "testSummary": { - "executables": [ - "TA", - "team1", - "team2", - "team3" - ], - "packages": [ - { - "count": 3, - "name": "TA" - }, - { - "count": 4, - "name": "team1" - }, - { - "count": 2, - "name": "team2" - }, - { - "count": 1, - "name": "team3" - }, - { - "count": 4, - "name": "timed_tests" - } - ] - }, - "title": "415 Grades" -} \ No newline at end of file diff --git a/tests/scripts/build.sh b/tests/scripts/build.sh new file mode 100755 index 00000000..d143c818 --- /dev/null +++ b/tests/scripts/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# script: build.sh +# author: justin meimar +# description: build each project in the current directory, track +# builds that fail in a file failed_builds.txt + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Set the partial config path to be absolute +failed_builds="$script_dir/failed_builds.txt" +> "$failed_builds" + +directories=($(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2)) + +for dir in "${directories[@]}"; do + if [ -d "$dir" ]; then + echo "-- Building project: ${dir}" + + cd "$dir" || { + echo "$dir: Failed to change directory" >> "$failed_builds" + continue + } + + # change into build directory + mkdir -p build + + # make VCalc + cd build || { + echo "$dir: Failed to change to build directory" >> "$failed_builds" + cd .. + continue + } + + if ! cmake ..; then + echo "$dir: cmake failed" >> "$failed_builds" + cd ../.. + continue + fi + + # Run make and handle failure + if ! make -j 2; then + echo "$dir: make failed" >> "$failed_builds" + fi + + # change back to previous directory for next iteration + cd ../.. + fi +done + +echo "Build process completed. Check $failed_builds for any failed builds." \ No newline at end of file diff --git a/tests/scripts/clean.sh b/tests/scripts/clean.sh new file mode 100755 index 00000000..d92c3000 --- /dev/null +++ b/tests/scripts/clean.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +directories=($(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2)) + +# echo ${directories[@]} + +for dir in "${directories[@]}"; do + + if [ -d "$dir" ]; then + + cd $dir + + echo "Cleaning ${dir}" + + # remove build directory + if [ -d "build" ]; then + rm -r "./build" + echo " -- rm build" + fi + + # remove generated antlr4 files + if [ -d "gen" ]; then + rm -r "./gen" + echo " -- rm gen" + fi + + # remove binary dir + if [ -d "bin" ]; then + rm -r "./bin" + echo " -- rm bin" + fi + + if [ -d "grammar/.antlr" ]; then + rm -r "./grammar/.antlr" + echo " -- rm .antlr" + fi + + cd - > /dev/null + fi +done \ No newline at end of file diff --git a/tests/scripts/convert_tests.sh b/tests/scripts/convert_tests.sh new file mode 100755 index 00000000..d6cdffad --- /dev/null +++ b/tests/scripts/convert_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# script: copy_tests.sh +# author: justin meimar +# description: convert old test formats from F23 into F24 tester format. +# + +# Get the absolute path of the script's directory +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +directories=($(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2)) + +if [ ! -d $testdir ]; then + echo "Could not find test directory." + exit 1 +fi + +for dir in "${directories[@]}"; do + # echo "$dir" + cd "$dir" + + convert_dir="${script_dir}/${dir}/tests" + + if [ -d "${script_dir}/${dir}/testfiles" ]; then + continue + fi + echo "${convert_dir}" + python "${script_dir}/convertFiles.py" "${convert_dir}" + + cd .. +done \ No newline at end of file diff --git a/tests/scripts/copy_tests.sh b/tests/scripts/copy_tests.sh new file mode 100755 index 00000000..e3e96350 --- /dev/null +++ b/tests/scripts/copy_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# script: copy_tests.sh +# author: justin meimar +# description: Copy test files into one large directory for running the grader. + +# Get the absolute path of the script's directory +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +testdir="testfiles" +rm -r "${script_dir}/${testdir}" + +directories=($(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2)) + +mkdir -p "$testdir" + +if [ ! -d $testdir ]; then + echo "Could not find test directory." + exit 1 +fi + +for dir in "${directories[@]}"; do + # echo "$dir" + cd "$dir" + + team_name=${dir#*-} + expected_test_dir="${script_dir}/${dir}/tests/testfiles/${team_name}" + + if [ -d "${expected_test_dir}" ]; then + echo "-- Found properly formatted testfiles ${team_name}" + echo " -- ${expected_test_dir}" + cp -r "${expected_test_dir}" "${script_dir}/${testdir}/" + + # cp -r "${expected_test_dir}" + else + echo "-- Bad test directory naming: ${team_name}" + + # CASE 1: the team did not wrap all their test in a self-named directory + # TODO: + + # CASE 2: the team tried to make a self-named directory, but failed. + # TODO: + fi + cd .. +done \ No newline at end of file diff --git a/tests/scripts/gen_config.sh b/tests/scripts/gen_config.sh new file mode 100755 index 00000000..36f327c2 --- /dev/null +++ b/tests/scripts/gen_config.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# script: gen_config.sh +# author: justin meimar +# description: Once every project is built, copy the paths of the binary and +# runtime into a config file. + +# Get the absolute path of the script's directory +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# Set the partial config path to be absolute +partial_config="$script_dir/partial_config.json" +> "$partial_config" + +directories=($(find . -maxdepth 1 -type d -exec basename {} \; | tail -n +2)) + +binary="vcalc" +shared_obj="libvcalcrt.so" + +# Echo the binary paths into the parital config +echo "{" >> "$partial_config" +echo "\"testedExecutablePaths\": {" >> "$partial_config" +for dir in "${directories[@]}"; do + full_dir="$script_dir/$dir" + team_name=${dir#*-} + if [ -d "$full_dir" ]; then + if [ -d "$full_dir/bin" ]; then + binary_path=$(find "$full_dir/bin" -name "$binary") + if [ -n "$binary_path" ]; then + echo " -- Copying binary from ${dir}" + echo " \"${team_name}\": \"$binary_path\"," >> "$partial_config" + else + echo " -- ERROR: Could not find binary for project: ${dir}" + fi + fi + fi +done + +# remove trailing comma on last binary entry +sed -i '$ s/,$//' "$partial_config" + +echo "-- Copied binary paths into partial config" + +echo "}," >> "$partial_config" + +# Echo the runtimes into the parital config +echo "\"runtimes\": {" >> "$partial_config" +for dir in "${directories[@]}"; do + full_dir="$script_dir/$dir" + team_name=${dir#*-} + if [ -d "$full_dir" ]; then + if [ -d "$full_dir/bin" ]; then + + so_path=$(find "$full_dir/bin" -name "$shared_obj") + + if [ -n "$binary_path" ] && [ -n "$so_path" ]; then + echo " -- Copying runtime from ${dir}" + echo " \"${team_name}\": \"$so_path\"," >> "$partial_config" + else + echo " -- ERROR: Could not find runtime for project: ${dir}" + fi + fi + fi +done + +# remove trailing comma on shared object entry +sed -i '$ s/,$//' "$partial_config" + +echo "}" >> "$partial_config" +echo "}" >> "$partial_config" + +echo "-- Copied runtime paths into partial config" \ No newline at end of file diff --git a/tests/scripts/grader.py b/tests/scripts/grader.py index 1168e577..c961713e 100644 --- a/tests/scripts/grader.py +++ b/tests/scripts/grader.py @@ -7,39 +7,20 @@ import json import pandas as pd from fractions import Fraction +from typing import List, Optional +from logging import Logger -MIN_COLUMNS=6 - -# score awarded to a team for passing all an attackers tests -DEFENSE_POINT_SCORE=2 -COHERENCE_POINT_SCORE=1 - -# Which package to use -TA_PACKAGE="gazprea-solution" - -GRADE_TIME=False - -# the test packages to use for timings -TIMED_PACKAGE="timed_tests" - -# the list of toolchains for which to record timings -TIMED_TOOLCHAIN="gazprea-opt" - -# the executable for which other executables should be compared too -TIMED_EXE_REFERENCE="TA" - -TIME_MAX_SECONDS=10 +MIN_COLUMNS = 6 +DEFENSE_POINT_SCORE = 2 +COHERENCE_POINT_SCORE = 1 +GRADE_TIME = False +TIME_MAX_SECONDS = 10 # weights TA_TEST_WEIGHT = 0.5 -COMPETATIVE_WEGIHT = 0.2 +COMPETATIVE_WEIGHT = 0.2 TIMING_WEIGHT = 0.1 -global data # the parsed JSON data -global OUTPUT_CSV # the filename of the output CSV -global n_attackers # how many test packages are in the tournament (gte n_defenders) -global n_defenders # how many exectuables are in the tournament (lte n_attackers) - from typing import List class Attack: @@ -111,6 +92,7 @@ def get_attack_header() -> pd.DataFrame: def get_timing_tests(toolchain) -> List[str]: timed_tests=[] + for defense_obj in toolchain["toolchainResults"]: for attack_obj in defense_obj["defenderResults"]: if attack_obj["attacker"] == TIMED_PACKAGE: @@ -134,6 +116,8 @@ def create_toolchain_summary_table(toolchains) -> pd.DataFrame: # normalize each value to compute the average tcs_table.iloc[1:, 1:] = tcs_table.iloc[1:, 1:] / len(toolchains) tcs_table.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") + print(f"============ TOOLCHAIN SUMMARY TABLE ============\n", tcs_table) + return tcs_table def create_toolchain_results_table(name, results) -> pd.DataFrame: @@ -142,7 +126,7 @@ def create_toolchain_results_table(name, results) -> pd.DataFrame: assert len(results) > 0, "Need at least one defending executable." df = get_attack_header() - + # each defense result represents all the attackers tests running on a single defending exe for defense_result in results: row = Defense(defense_result) row_df = row.get_competative_df() @@ -167,6 +151,8 @@ def create_toolchain_results_table(name, results) -> pd.DataFrame: # sum total competative points points_df.iloc[3, 1:] = points_df.iloc[:3, 1:].sum() df = pd.concat([df, points_df], ignore_index=True) + + print(f"============ TOOLCHAIN RESULT TABLE ({name}) ============\n", df) df.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") return df @@ -176,7 +162,8 @@ def create_timing_table(timed_toolchain): Create a table with a column for each tested executable and a row for each test in the testpackage(s) for which timing is desired. """ - timed_tests = get_timing_tests(timed_toolchain) + timed_tests = get_timing_tests(timed_toolchain) + assert len(timed_tests) > 0, "Found timed tests package with 0 tests." timing_df = pd.DataFrame( None, index=range(0, len(timed_tests)+1), columns=range(n_attackers)) @@ -189,6 +176,7 @@ def create_timing_table(timed_toolchain): timing_df.iloc[1:,1:] = timing_df.iloc[1:,1:].fillna(0).round(3) timing_df.to_csv(OUTPUT_CSV, index=False, header=False, mode='a') + print("============ TIMING TABLE ============\n", timing_df) insert_blank_row() # compute the relative timings as normalized by the fastest executable @@ -206,6 +194,7 @@ def create_timing_table(timed_toolchain): # append the total row to the relative timing row rel_timing_df = pd.concat([rel_timing_df, rel_total], ignore_index=True) rel_timing_df.to_csv(OUTPUT_CSV, index=False, header=False, mode='a') + print("============ RELATIVE TIMING TABLE ============\n", rel_timing_df) return rel_timing_df def create_test_summary_table(): @@ -216,13 +205,16 @@ def create_test_summary_table(): df.iloc[:3,0] = ["Test Summary", "Team Name", "Test Count"] # record the count and name of each test package pkgs = get_attacking_packages() + for i, package in enumerate(pkgs): df.at[1, i+1] = package["name"] df.at[2, i+1] = package["count"] - + print("============ SUMMARY TABLE ============\n", df) df.to_csv(OUTPUT_CSV, index=False, header=False, mode='a') -def create_final_summary_table(toolchain_summary, timing_summary) -> pd.DataFrame: +def create_final_summary_table( + toolchain_summary: pd.DataFrame, + timing_summary: Optional[pd.DataFrame]) -> pd.DataFrame: """ Create a final summary table and return a dataframe """ @@ -232,8 +224,7 @@ def create_final_summary_table(toolchain_summary, timing_summary) -> pd.DataFram "Code Style (10%)", "Final Grade (100%)" ] # Get Pass Rate on TA tests. - first_row = toolchain_summary.iloc[0] # get first row - index = fst.loc[first_row.str.contains(TA_PACKAGE, na=False)].index + index = get_attacking_package_names().index(TA_PACKAGE) ta_pass_rate_col = toolchain_summary.iloc[1:n_attackers, index] fst.iloc[0,1:n_attackers] = (ta_pass_rate_col.T * TA_TEST_WEIGHT).fillna(0).round(5) @@ -241,14 +232,16 @@ def create_final_summary_table(toolchain_summary, timing_summary) -> pd.DataFram comp_row = toolchain_summary.iloc[n_defenders+4, 1:(1+n_defenders)] max_comp_score = comp_row.max() normalized_comp_row = comp_row / max_comp_score - fst.iloc[1,1:n_attackers] = (normalized_comp_row * COMPETATIVE_WEGIHT).fillna(0).round(5) + fst.iloc[1,1:n_attackers] = (normalized_comp_row * COMPETATIVE_WEIGHT).fillna(0).round(5) # Get timing scores - timing_scores = (timing_summary.iloc[-1,1:] * TIMING_WEIGHT).fillna(0).round(5) - fst.iloc[2,1:n_attackers] = timing_scores - print(timing_scores) + if timing_summary is not None: + timing_scores = (timing_summary.iloc[-1,1:] * TIMING_WEIGHT).fillna(0).round(5) + fst.iloc[2,1:n_attackers] = timing_scores + print(timing_scores) # Write to CSV + print("============ FINAL SUMMARY TABLE ============\n", fst) fst.to_csv(OUTPUT_CSV, index=False, header=False, mode='a') return fst @@ -273,14 +266,22 @@ def fill_csv(): insert_blank_row() ## STEP 4: timing results - timed_toolchain = [tc for tc in data["results"] if tc["toolchain"] == "gazprea"] - assert len(timed_toolchain), f"Could not find the toolchain supposed to be timed: {TIMED_TOOLCHAIN}" - rel_timing_table = create_timing_table(timed_toolchain[0]) - insert_blank_row() + if is_timed_grading(): + timed_toolchain = [tc for tc in data["results"] if tc["toolchain"] == TIMED_TOOLCHAIN] + assert len(timed_toolchain), f"Could not find the toolchain supposed to be timed: {TIMED_TOOLCHAIN}" + rel_timing_table = create_timing_table(timed_toolchain[0]) + insert_blank_row() ## STEP 5: final summary and grades create_final_summary_table(tcs, rel_timing_table) +def is_timed_grading(): + """ + We include a timing table to the final grades if the necessary variables are supplied via + the CLI arguments. + """ + return True if all([TIMED_PACKAGE, TIMED_EXE_REFERENCE, TIMED_TOOLCHAIN]) else False + def get_attacking_packages(): """ Returns a list of all attacking packages, sorted by those packages for which @@ -288,13 +289,17 @@ def get_attacking_packages(): columns and rows in the sheet. """ priority_list = get_defending_executables() - return sorted( + attacking_pkgs = sorted( data["testSummary"]["packages"], - key=lambda exe: (exe["name"] not in priority_list) - ) + key=lambda exe: (exe["name"] not in priority_list, exe["name"]) + ) + + return attacking_pkgs def get_attacking_package_names(): - return [ pkg["name"] for pkg in get_attacking_packages() ] + attacking_packag_names = [ pkg["name"] for pkg in get_attacking_packages() ] + + return attacking_packag_names def get_defending_executables(): return sorted(data["testSummary"]["executables"]) @@ -305,14 +310,13 @@ def get_student_packages(): student_packages = [ pkg for pkg in all_packages if pkg in student_exes] return student_packages -if __name__ == "__main__": - +def parse_arguments(): parser = argparse.ArgumentParser(description='Produce Grade CSV based on JSON input.') # Required arguments parser.add_argument('json_file', type=str, help='Path to the JSON file') parser.add_argument('-o', '--output', type=str, required=True, help='Path to the output CSV file') - parser.add_argument('--ta-package', type=str, required=True, default="", help='The test packaged used for TA tests.') + parser.add_argument('--ta-package', type=str, required=True, help='The test package used for TA tests.') # Optional timed arguments group timed_group = parser.add_argument_group('timed options') @@ -327,23 +331,34 @@ def get_student_packages(): if any(timed_args) and not all(timed_args): parser.error("All timed arguments (--timed-package, --timed-toolchain, --timed-exe-reference) must be provided together") - # initialize global parameters - JSON_FILE = args.json_file - OUTPUT_CSV = args.output - TA_PACKAGE = args.ta_package - if all(timed_args): + return args + +def main(): + args = parse_arguments() + + global data, OUTPUT_CSV, n_attackers, n_defenders + global TA_PACKAGE, TIMED_PACKAGE, TIMED_TOOLCHAIN, TIMED_EXE_REFERENCE + + # Initialize global parameters + OUTPUT_CSV = args.output + TA_PACKAGE = args.ta_package + + if all([args.timed_package, args.timed_toolchain, args.timed_exe_reference]): TIMED_PACKAGE = args.timed_package TIMED_TOOLCHAIN = args.timed_toolchain TIMED_EXE_REFERENCE = args.timed_exe_reference - # init globals - with open(JSON_FILE, "r") as file: + # Initialize data + with open(args.json_file, "r") as file: data = json.load(file) - + n_attackers = len(get_attacking_packages()) n_defenders = len(get_defending_executables()) with open(OUTPUT_CSV, "w") as csv: fill_csv() df = pd.read_csv(OUTPUT_CSV) - print(df.to_string()) \ No newline at end of file + print(df.to_string()) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test b/tests/test deleted file mode 100755 index 5146c1f2..00000000 Binary files a/tests/test and /dev/null differ diff --git a/tests/test.o b/tests/test.o deleted file mode 100755 index 1ed3a559..00000000 Binary files a/tests/test.o and /dev/null differ