diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp index 7c715ddb..61602c3a 100644 --- a/src/analysis/Grader.cpp +++ b/src/analysis/Grader.cpp @@ -2,15 +2,6 @@ #include #include -// File static namespace. -namespace { - -bool containsString(const std::vector& vec, const std::string& str) { - return std::find(vec.begin(), vec.end(), str) != vec.end(); -} - -} - namespace tester { void Grader::buildResults() { @@ -18,33 +9,33 @@ void Grader::buildResults() { outputJson["title"] = "415 Grades"; outputJson["results"] = JSON::array(); - // infer the name of each time from the executable string + JSON testSummary = { + {"packages", JSON::array()}, + {"executables", JSON::array()} + }; + for (const auto& exe : cfg.getExecutables()) { - std::cout << "exe: " << exe.first << std::endl; - defendingExes.push_back(exe.first); + + std::string exeName = exe.first; + defendingExes.push_back(exeName); + testSummary["executables"].push_back(exeName); } - - // build a preliminary test summary object - JSON testSummary = JSON::array(); - attackingTestPackages = defendingExes; // start with a copy to preserve symmetric ordering + for (const auto& testPackage : testSet) { std::string packageName = testPackage.first; + attackingTestPackages.push_back(packageName); - if (!containsString(defendingExes, packageName)) { - attackingTestPackages.push_back(packageName); - } - - // Create the summary item. - JSON summaryItem = {{"team", packageName}}; - size_t count = 0; - for (const auto& subpackage : testPackage.second) { + int count = 0; + for (const auto &subpackage : testPackage.second) count += subpackage.second.size(); - } - summaryItem["testCount"] = count; - testSummary.push_back(summaryItem); + + JSON packageSummary = { + {"name", packageName}, + {"count", count} + }; + testSummary["packages"].push_back(packageSummary); } - outputJson["testSummary"] = testSummary; // Start running tests. Make a pass rate table for each toolchain. @@ -68,12 +59,20 @@ void Grader::buildResults() { else tc.setTestedRuntime(""); + // Find max string length of team name for formatting stdout + auto maxNameLength = static_cast(std::max_element( + attackingTestPackages.begin(), attackingTestPackages.end(), + [](const std::string &a, const std::string &b) { + return a < b; + } + )->size()); + // Iterate over attackers. - int maxNameLength = 20, arrowStart = 10; for (const std::string& attacker : attackingTestPackages) { - std::cout << " " << std::setw(arrowStart) << std::left << "(" + attacker + ")" - << std::setw(maxNameLength - arrowStart) << " --> " - << std::setw(10) << "(" + defender + ") "; + + std::cout << " " << std::left << std::setw(maxNameLength + 2) << ("(" + attacker + ")") // +2 for the parentheses + << " --> " + << std::left << std::setw(maxNameLength + 2) << ("(" + defender + ")"); JSON attackResults = {{"attacker", attacker}, {"timings", JSON::array()}}; diff --git a/tests/multi_exe_tests/Config.json b/tests/multi_exe_tests/Config.json new file mode 100644 index 00000000..888724be --- /dev/null +++ b/tests/multi_exe_tests/Config.json @@ -0,0 +1,57 @@ +{ + "testDir": "tests", + "testedExecutablePaths": { + "test": "/usr/bin/clang" + }, + "toolchains": { + "LLVM": [ + { + "stepName": "compile", + "executablePath": "$EXE", + "arguments": [ + "$INPUT", + "-Wno-everything", + "-fno-show-column", + "-fno-show-source-location", + "-o", "$OUTPUT" + ], + "output": "test", + "allowError": true + }, + { + "stepName": "run", + "executablePath": "$INPUT", + "arguments": [], + "output": "-", + "usesInStr": true, + "allowError": true + } + ], + "LLVM-opt": [ + { + "stepName": "compile", + "executablePath": "$EXE", + "arguments": [ + "$INPUT", + "-O3", + "-Wno-everything", + "-fno-show-column", + "-fno-show-source-location", + "-o", "$OUTPUT" + ], + "output": "test", + "allowError": true + }, + { + "stepName": "run", + "executablePath": "$INPUT", + "arguments": [], + "output": "-", + "usesInStr": true, + "allowError": true + } + ] + } +} + + diff --git a/tests/multi_exe_tests/GradeConfig.json b/tests/multi_exe_tests/GradeConfig.json index faa73632..ad82b579 100644 --- a/tests/multi_exe_tests/GradeConfig.json +++ b/tests/multi_exe_tests/GradeConfig.json @@ -3,7 +3,8 @@ "testedExecutablePaths": { "team1": "/usr/bin/clang", "team2": "/usr/bin/clang", - "team3": "/usr/bin/clang" + "team3": "/usr/bin/clang", + "TA": "/usr/bin/clang" }, "toolchains": { "LLVM": [ diff --git a/tests/multi_exe_tests/results.json b/tests/multi_exe_tests/results.json new file mode 100644 index 00000000..f42a90e0 --- /dev/null +++ b/tests/multi_exe_tests/results.json @@ -0,0 +1,813 @@ +{ + "results": [ + { + "toolchain": "LLVM", + "toolchainResults": [ + { + "defender": "TA", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.01034884 + ], + [ + "001_ta.c", + 0.010497002 + ], + [ + "000_ta.c", + 0.01060118 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010512948 + ], + [ + "002.c", + 0.010659881 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010635542 + ], + [ + "001_mismatch.c", + 0.010649832 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.003260794 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010497766 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.021130683 + ] + ] + } + ] + }, + { + "defender": "team1", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010356955 + ], + [ + "001_ta.c", + 0.010690674 + ], + [ + "000_ta.c", + 0.010358642 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010400347 + ], + [ + "002.c", + 0.010691282 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010521112000000001 + ], + [ + "001_mismatch.c", + 0.010687718 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.003111213 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010650402 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.023056491 + ] + ] + } + ] + }, + { + "defender": "team2", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010609016 + ], + [ + "001_ta.c", + 0.010618255 + ], + [ + "000_ta.c", + 0.010713738 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010620003 + ], + [ + "002.c", + 0.010617953 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010505006 + ], + [ + "001_mismatch.c", + 0.010492173 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.002934806 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010501177 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.018494322 + ] + ] + } + ] + }, + { + "defender": "team3", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010543416 + ], + [ + "001_ta.c", + 0.010498105 + ], + [ + "000_ta.c", + 0.010456271 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010723391 + ], + [ + "002.c", + 0.010635687 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010491462 + ], + [ + "001_mismatch.c", + 0.010612427 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.005150645 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010325688 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.020746287 + ] + ] + } + ] + } + ] + }, + { + "toolchain": "LLVM-opt", + "toolchainResults": [ + { + "defender": "TA", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010589115 + ], + [ + "001_ta.c", + 0.010642028 + ], + [ + "000_ta.c", + 0.010602248 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.01048172 + ], + [ + "002.c", + 0.010688037 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010624927 + ], + [ + "001_mismatch.c", + 0.010670567 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.005373263 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010646021 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.008123031 + ] + ] + } + ] + }, + { + "defender": "team1", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010668507 + ], + [ + "001_ta.c", + 0.010632455 + ], + [ + "000_ta.c", + 0.010294811 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010468743 + ], + [ + "002.c", + 0.010631441 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010672274 + ], + [ + "001_mismatch.c", + 0.01061128 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.001949765 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010672657 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.003749667 + ] + ] + } + ] + }, + { + "defender": "team2", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010509623 + ], + [ + "001_ta.c", + 0.010493248 + ], + [ + "000_ta.c", + 0.010390238 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010624393 + ], + [ + "002.c", + 0.010722831 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.01037731 + ], + [ + "001_mismatch.c", + 0.010722518 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.004814753 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010687889 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.010968341 + ] + ] + } + ] + }, + { + "defender": "team3", + "defenderResults": [ + { + "attacker": "TA", + "passCount": 3, + "testCount": 3, + "timings": [ + [ + "002_ta.c", + 0.010670114 + ], + [ + "001_ta.c", + 0.010697331 + ], + [ + "000_ta.c", + 0.010348531 + ] + ] + }, + { + "attacker": "team1", + "passCount": 3, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010680346 + ], + [ + "002.c", + 0.01049487 + ], + [ + "003_timeout.c", + 0.0 + ] + ] + }, + { + "attacker": "team2", + "passCount": 1, + "testCount": 2, + "timings": [ + [ + "002.c", + 0.010588078 + ], + [ + "001_mismatch.c", + 0.010575755 + ] + ] + }, + { + "attacker": "team3", + "passCount": 1, + "testCount": 1, + "timings": [ + [ + "001.c", + 1.008140228 + ] + ] + }, + { + "attacker": "timed_tests", + "passCount": 4, + "testCount": 4, + "timings": [ + [ + "004_error.c", + 0.0 + ], + [ + "001.c", + 0.010635711 + ], + [ + "002.c", + 0.0 + ], + [ + "003_timeout.c", + 2.006529471 + ] + ] + } + ] + } + ] + } + ], + "testSummary": { + "executables": [ + "TA", + "team1", + "team2", + "team3" + ], + "packages": [ + { + "count": 3, + "name": "TA" + }, + { + "count": 4, + "name": "team1" + }, + { + "count": 2, + "name": "team2" + }, + { + "count": 1, + "name": "team3" + }, + { + "count": 4, + "name": "timed_tests" + } + ] + }, + "title": "415 Grades" +} \ No newline at end of file diff --git a/tests/multi_exe_tests/tests/timed_tests/001.c b/tests/multi_exe_tests/tests/timed_tests/001.c index be7958b8..66c999e6 100644 --- a/tests/multi_exe_tests/tests/timed_tests/001.c +++ b/tests/multi_exe_tests/tests/timed_tests/001.c @@ -11,6 +11,11 @@ int main() { printf("%c%c%c", c[2], c[1], c[0]); + int i = 0; + for (int i = 0; i < 1000000; i++) { + i++; + } + return 0; } diff --git a/tests/multi_exe_tests/tests/timed_tests/002.c b/tests/multi_exe_tests/tests/timed_tests/002.c index a3f5b64a..406de5d0 100644 --- a/tests/multi_exe_tests/tests/timed_tests/002.c +++ b/tests/multi_exe_tests/tests/timed_tests/002.c @@ -1,14 +1,21 @@ #include -#define ITER 100000000 +#define iter 100000000 int main() { int j = 0; - for(int i = 0; i < ITER; i++) { - if (j + i -1 < 0xFFFF) { + for(int i = 0; i < iter; i++) { + if (j + i -1 < 0xffff) { j += i; } } + sleep(1); + + int i = 0; + for (int i = 0; i < 1000000; i++) { + i++; + } + return (j < 0xffffffff); } \ No newline at end of file diff --git a/tests/multi_exe_tests/tests/timed_tests/003_timeout.c b/tests/multi_exe_tests/tests/timed_tests/003_timeout.c index 72832ac9..29af9b20 100644 --- a/tests/multi_exe_tests/tests/timed_tests/003_timeout.c +++ b/tests/multi_exe_tests/tests/timed_tests/003_timeout.c @@ -8,5 +8,15 @@ int main() { } } - return (j < 0xffffffff); -} \ No newline at end of file + int i = 0; + for (int i = 0; i < 10000000; i++) { + i++; + } + + sleep(2); + + printf("123"); + return 0; +} + +// CHECK:123 \ No newline at end of file diff --git a/tests/multi_exe_tests/tests/timed_tests/004_error.c b/tests/multi_exe_tests/tests/timed_tests/004_error.c index a6a079e6..d006ce6d 100644 --- a/tests/multi_exe_tests/tests/timed_tests/004_error.c +++ b/tests/multi_exe_tests/tests/timed_tests/004_error.c @@ -16,7 +16,7 @@ int main() { fprintf(stderr, "DivideByZeroError: a was about to be divided by 0!"); exit(EXIT_DIVIDE_BY_ZERO); } - + return 0; } diff --git a/tests/scripts/grade.py b/tests/scripts/grade.py index 9667531e..18432fc3 100644 --- a/tests/scripts/grade.py +++ b/tests/scripts/grade.py @@ -6,14 +6,35 @@ import argparse import json import pandas as pd -import numpy as np from fractions import Fraction -from typing import Dict, List, Tuple, Any +# output file OUTPUT_CSV="output.csv" -MIN_COLUMNS=7 + +# width of the CSV for initialization +MIN_COLUMNS=6 + +# score awarded to a team for passing all an attackers tests DEFENSE_POINT_SCORE=2 +# the test packages to use for timings +TIMED_PACKAGES=[ + "timed_tests" +] + +# the list of toolchains for which to record timings +TIMED_TOOLCHAINS=[ + "LLVM-opt" +] + +# the executable for which other executables should be compared too +TIMED_EXE_REFERENCE="TA" + +# weights +TA_TEST_WEIGHT = 0.5 +COMPETATIVE_WEGIHT = 0.2 +TIMING_WEIGHT = 0.1 + def create_competative_table(toolchain_results, n_exe, n_tests): """ Generate a table for the pass rate. @@ -77,9 +98,9 @@ def create_test_summary_table(data) -> pd.DataFrame: df.at[1,0] = "Team Name" df.at[2, 0] = "Test Count" - for i, team in enumerate(summary): - df.at[1, i+1] = team["team"] - df.at[2, i+1] = team["testCount"] + for i, package in enumerate(summary["packages"]): + df.at[1, i+1] = package["name"] + df.at[2, i+1] = package["count"] return df @@ -97,49 +118,120 @@ def create_toolchain_summary_table(toolchains, n_teams) -> pd.DataFrame: avg_score = avg_score / len(toolchains) tcs_table.at[d+1, a+1] = avg_score + tcs_table.at[n_teams+4, 0] = "total competative points" for team_idx in range(0, n_teams): - offensive_avg = 0 - defensive_avg = 0 - coherence_avg = 0 + offensive_sum = 0 + defensive_sum = 0 + coherence_sum = 0 for toolchain in toolchains: - offensive_avg += Fraction(toolchain.at[n_teams+1, team_idx+1]) - defensive_avg += Fraction(toolchain.at[n_teams+2, team_idx+1]) - coherence_avg += Fraction(toolchain.at[n_teams+3, team_idx+1]) + offensive_sum += Fraction(toolchain.at[n_teams+1, team_idx+1]) + defensive_sum += Fraction(toolchain.at[n_teams+2, team_idx+1]) + coherence_sum += Fraction(toolchain.at[n_teams+3, team_idx+1]) - tcs_table.at[n_teams+1, team_idx+1] = offensive_avg / len(toolchains) - tcs_table.at[n_teams+2, team_idx+1] = defensive_avg / len(toolchains) - tcs_table.at[n_teams+3, team_idx+1] = coherence_avg / len(toolchains) + offensive_avg = offensive_sum/ len(toolchains) + defensive_avg = defensive_sum/ len(toolchains) + coherence_avg = coherence_sum/ len(toolchains) + + tcs_table.at[n_teams+1, team_idx+1] = offensive_avg + tcs_table.at[n_teams+2, team_idx+1] = defensive_avg + tcs_table.at[n_teams+3, team_idx+1] = coherence_avg + + # sum all the averages + tcs_table.at[n_teams+4, team_idx+1] = offensive_avg + defensive_avg + coherence_avg + + # calculate the competative score as a ratio over the max + tcs_table.at[n_teams+5, 0] = "competative score" + max_competative = tcs_table.iloc[n_teams+4, 1:n_teams+1].max() + tcs_table.iloc[n_teams+5, 1:n_teams+1] \ + = (tcs_table.iloc[n_teams+4, 1:n_teams+1] / max_competative) + + # calcaulte pass rate for TA tests + tcs_table.at[n_teams+6, 0] = "TA test score" + tcs_table.iloc[n_teams+6, 1:n_teams+1] = tcs_table.iloc[1:n_teams+1, 1].T return tcs_table -def create_final_summary_table(data) -> pd.DataFrame: +def create_execution_timing_table(results, n_exe, n_timed_tests): """ - Create a final summary table and return a dataframe + Create a table with a column for each tested executable and a row for each test + in the testpackage(s) for which timing is desired. """ - # summary = data["testSummary"] - df = pd.DataFrame(None, index=range(3), columns=range(3)) + df = pd.DataFrame(None, index=range(n_timed_tests + 2), columns=range(n_exe)) + + for j, defense_obj in enumerate(results): + df.at[0, j+1] = defense_obj["defender"] + for attack_obj in defense_obj["defenderResults"]: + if attack_obj["attacker"] in TIMED_PACKAGES: + for i, test in enumerate(attack_obj["timings"]): + df.at[i+1, j+1] = round(test[1], 5) # time of test execution (s) + df.at[i+1, 0] = test[0] # name of test + + df.at[n_timed_tests+1, 0] = "Speedup Points" + + # Calculate speedup points + for j in range(0, n_exe): + speedup_points = 0.0 + for i in range(n_timed_tests): + ta_time = df.at[i+1, 1] # TA execution time + team_time = df.at[i+1, j+1] # Team execution time + + if pd.notna(ta_time) and pd.notna(team_time) and ta_time != 0: + speedup_points += ta_time / team_time + + df.at[n_timed_tests+1, j+1] = round(speedup_points, 5) + + # calculate the competative score as a ratio over the max + df.at[n_timed_tests+2, 0] = "relative timing score" + max_timing_score = df.iloc[n_timed_tests+1, 1:n_exe+1].max() + + df.iloc[n_timed_tests + 2, 1:n_exe + 1] = ( + df.iloc[n_timed_tests + 1, 1:n_exe + 1] / max_timing_score + ).fillna(0).round(4) + return df -def create_execution_timing_table(results, toolchain, testpackage, n_exe): - # df = pd.DataFrame(None, index=range(len(testsuite)+1), columns=range(n_teams)) - print("toolchain: ", toolchain) - print("testpackage: ", testpackage) - # print(results) +def create_final_summary_table(toolchain_summary, timing_summary, n_exe) -> pd.DataFrame: + """ + Create a final summary table and return a dataframe + """ + df = pd.DataFrame(None, index=range(3), columns=range(n_exe+1)) + + df.at[0, 0] = "TA Testing (50%)" + df.at[1, 0] = "Competative Testing (20%)" + df.at[2, 0] = "Timing Testing (10%)" + df.at[3, 0] = "Grammar (10%)" + df.at[4, 0] = "Code Style (10%)" + df.at[5, 0] = "Final Grade (100%)" + + df.iloc[0, 1:n_exe+1] = ( + toolchain_summary.iloc[10, 1:n_exe+1] * TA_TEST_WEIGHT + ).fillna(0).round(4) + + df.iloc[1, 1:n_exe+1] = ( + toolchain_summary.iloc[9, 1:n_exe+1] * COMPETATIVE_WEGIHT + ).fillna(0).round(4) + + df.iloc[2, 1:n_exe+1] = ( + timing_summary.iloc[5, 1:n_exe+1] * TIMING_WEIGHT + ).fillna(0).round(4) - for defense_obj in results: - print(defense_obj["defender"]) - for attack_obj in defense_obj["defenderResults"]: - if attack_obj["attacker"] == testpackage: - # print(attack_obj) - for test in attack_obj["timings"]: - print(test) + df.iloc[3, 1:n_exe + 1] = 0 + df.iloc[4, 1:n_exe + 1] = 0 + + df.iloc[5, 1:n_exe + 1] = df.iloc[0:5, 1:n_exe + 1].sum(axis=0) + return df def generate_csv(data): ts_table = create_test_summary_table(data) ts_table.to_csv(OUTPUT_CSV, index=False, header=False) insert_blank_row() - + + # how many tests are timed + n_timed_tests = next( + (pkg for pkg in data["testSummary"]["packages"] if pkg['name'] in TIMED_PACKAGES), None + )['count'] + toolchain_tables = [] time_tables = [] for result in data["results"]: @@ -149,17 +241,23 @@ def generate_csv(data): n_exe = len(toolchain_results) n_tests = len(toolchain_results[0]["defenderResults"]) - tc_table = create_competative_table(toolchain_results, n_exe, n_tests) - time_table = create_execution_timing_table(toolchain_results, toolchain_name, "timed_tests", n_exe) - + tc_table = create_competative_table(toolchain_results, n_exe, n_tests) toolchain_tables.append(tc_table) - time_tables.append(time_table) + # write the toolchain table to the CSV insert_label_row(toolchain_name) tc_table.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") - insert_blank_row() - + + if toolchain_name in TIMED_TOOLCHAINS: + time_table = create_execution_timing_table(toolchain_results, n_exe, n_timed_tests) + time_tables.append(time_table) + + # write the timing table to the CSV + insert_label_row(f"{toolchain_name} Timings") + time_table.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") + insert_blank_row() + # toolchain summay table insert_label_row("Toolchain Summary") tcs_table = create_toolchain_summary_table(toolchain_tables, n_exe) @@ -167,7 +265,9 @@ def generate_csv(data): insert_blank_row() # final summary table - fs_table = create_final_summary_table(tcs_table) + + insert_label_row("Final Grade Summary") + fs_table = create_final_summary_table(tcs_table, time_tables[0], n_exe) fs_table.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") return OUTPUT_CSV @@ -187,6 +287,36 @@ def insert_blank_row(): df = pd.DataFrame(None, index=range(1), columns=range(MIN_COLUMNS)) df.to_csv(OUTPUT_CSV, index=False, header=False, mode="a") +def sort_results(data): + """ + It is necessary for the order of attacking test packages to "line up" with the executables. + Specifically, the packages for which a corresponding executable exists must be ordered + first. + """ + summary = data["testSummary"] + executables = summary["executables"] + executable_set = set(executables) + executable_index = {name: idx for idx, name in enumerate(executables)} + + # sort the packages in the summary to be executables first + summary["packages"] = sorted( + summary["packages"], + key=lambda pkg: ( + pkg["name"] not in executable_set, + executable_index.get(pkg["name"], float('inf')) + ) + ) + + # sort the attack order to be packages with corresponding executables first. + for toolchain_result in data["results"]: + for defender_result in toolchain_result["toolchainResults"]: + defender_result["defenderResults"] = sorted( + defender_result["defenderResults"], + key=lambda res: (res["attacker"] not in executable_set, executable_index.get(res["attacker"], float('inf'))) + ) + + return data + if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -199,7 +329,7 @@ def insert_blank_row(): with open(args.file, "r") as file: data = json.load(file) - grade_csv = generate_csv(data) + grade_csv = generate_csv(sort_results(data)) # reload the csv for printing df = pd.read_csv(grade_csv)