From f6852736ee0eeafce2adadf22d7c216ba1b7a622 Mon Sep 17 00:00:00 2001 From: Justin Meimar Date: Tue, 28 May 2024 13:52:53 -0600 Subject: [PATCH] Passing all Blahaj-Wizards tests in converted test format --- include/Colors.h | 13 + include/analysis/Grader.h | 126 +- include/csv/csv.h | 1199 ----------------- include/testharness/TestHarness.h | 1 + include/tests/TestFile.h | 5 + include/tests/TestParser.h | 3 + include/toolchain/Command.h | 5 +- src/analysis/Grader.cpp | 60 +- src/main.cpp | 4 +- src/testharness/TestHarness.cpp | 5 +- src/tests/TestParser.cpp | 51 +- src/tests/TestRunning.cpp | 253 +++- src/toolchain/Command.cpp | 1 + src/toolchain/ToolChain.cpp | 1 - tests/grade_tests/team1/001.c | 8 +- tests/grade_tests/team3/002.c | 3 - tests/grades.json | 70 + tests/run_grader_tests.sh | 14 +- tests/user_tests/comments/002_comment_block.c | 2 +- tests/user_tests/io/002.out | 4 + tests/user_tests/io/002_input_multi.c | 26 +- tests/user_tests/io/003_input_file.c | 2 +- tests/user_tests/io/a.out | Bin 0 -> 16072 bytes 23 files changed, 505 insertions(+), 1351 deletions(-) create mode 100644 include/Colors.h delete mode 100644 include/csv/csv.h delete mode 100644 tests/grade_tests/team3/002.c create mode 100644 tests/grades.json create mode 100644 tests/user_tests/io/002.out create mode 100755 tests/user_tests/io/a.out diff --git a/include/Colors.h b/include/Colors.h new file mode 100644 index 00000000..d0779eed --- /dev/null +++ b/include/Colors.h @@ -0,0 +1,13 @@ +#ifndef COLORS_H +#define COLORS_H + +#include + +namespace Colors { + + inline const std::string GREEN = "\033[32m"; + inline const std::string RED = "\033[31m"; + inline const std::string RESET = "\033[0m"; +} + +#endif \ No newline at end of file diff --git a/include/analysis/Grader.h b/include/analysis/Grader.h index 4d2e6886..e19dca9c 100644 --- a/include/analysis/Grader.h +++ b/include/analysis/Grader.h @@ -3,14 +3,19 @@ #include "config/Config.h" #include "tests/Util.h" -#include "csv/csv.h" +#include "json.hpp" #include #include #include +#include + +using JSON = nlohmann::json; namespace tester { +// typedef std::map + class Grader { public: // No default constructor. @@ -19,7 +24,10 @@ class Grader { // Construct with output file path. explicit Grader(const Config &cfg); - // void dump(std::ostream &os) const { analysis.dumpSV(os); } + void dump(std::ostream &os) const { + std::string jsonString = results.dump(2); + os << jsonString; + } private: // Build the results to produce our sheet. @@ -36,9 +44,121 @@ class Grader { TestModule tests; // The filtered (must have exe and tests) names of all solutions that will be tested. - std::vector names; + std::vector names; + + // JSON object + JSON results; + }; } // End namespace tester #endif //TESTER_GRADER_H + + + +/** +The Vision: +[ + { + "toolchain": "arm", + "results": { + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 3, + "timings (ms)": [10.6, 90.3, 8.5] + }, + { + "atacker": "team1", + "defender": "team2", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [10.6, -1, -1] + }, + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [-1, 4, -1] + }, + { + "atacker": "team2", + "defender": "team1", + "testCount": 50, + "testPassed": 48, + "timings (ms)": [10.6, 90.3, 8.5...............] + }, + ...... + } + }, { + "toolchain": "riscv", + "results": { + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 3, + "timings (ms)": [10.6, 90.3, 8.5] + }, + { + "atacker": "team1", + "defender": "team2", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [10.6, -1, -1] + }, + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [-1, 4, -1] + }, + { + "atacker": "team2", + "defender": "team1", + "testCount": 50, + "testPassed": 48, + "timings (ms)": [10.6, 90.3, 8.5...............] + }, + ...... + } + }, { + "toolchain": "riscv", + "results": { + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 3, + "timings (ms)": [10.6, 90.3, 8.5] + }, + { + "atacker": "team1", + "defender": "team2", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [10.6, -1, -1] + }, + { + "atacker": "team1", + "defender": "team1", + "testCount": 3, + "testPassed": 1, + "timings (ms)": [-1, 4, -1] + }, + { + "atacker": "team2", + "defender": "team1", + "testCount": 50, + "testPassed": 48, + "timings (ms)": [10.6, 90.3, 8.5...............] + }, + ...... + } + } +] +*/ \ No newline at end of file diff --git a/include/csv/csv.h b/include/csv/csv.h deleted file mode 100644 index e227de3f..00000000 --- a/include/csv/csv.h +++ /dev/null @@ -1,1199 +0,0 @@ -// Copyright: (2012-2015) Ben Strasser -// License: BSD-3 -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors -// may be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. - -#ifndef CSV_H -#define CSV_H - -#include -#include -#include -#include -#include -#include -#include -#ifndef CSV_IO_NO_THREAD -#include -#include -#include -#endif -#include -#include -#include -#include -#include - -namespace io { -//////////////////////////////////////////////////////////////////////////// -// LineReader // -//////////////////////////////////////////////////////////////////////////// - -namespace error { -struct base : std::exception { - virtual void format_error_message() const = 0; - - const char *what() const noexcept override { - format_error_message(); - return error_message_buffer; - } - - mutable char error_message_buffer[2048]; -}; - -// this only affects the file name in the error message -const int max_file_name_length = 1024; - -struct with_file_name { - with_file_name() { std::memset(file_name, 0, sizeof(file_name)); } - - void set_file_name(const char *file_name) { - if (file_name != nullptr) { - // This call to strncpy has parenthesis around it - // to silence the GCC -Wstringop-truncation warning - (strncpy(this->file_name, file_name, sizeof(this->file_name))); - this->file_name[sizeof(this->file_name) - 1] = '\0'; - } else { - this->file_name[0] = '\0'; - } - } - - char file_name[max_file_name_length + 1]; -}; - -struct with_file_line { - with_file_line() { file_line = -1; } - - void set_file_line(int file_line) { this->file_line = file_line; } - - int file_line; -}; - -struct with_errno { - with_errno() { errno_value = 0; } - - void set_errno(int errno_value) { this->errno_value = errno_value; } - - int errno_value; -}; - -struct can_not_open_file : base, with_file_name, with_errno { - void format_error_message() const override { - if (errno_value != 0) - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Can not open file \"%s\" because \"%s\".", file_name, - std::strerror(errno_value)); - else - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Can not open file \"%s\".", file_name); - } -}; - -struct line_length_limit_exceeded : base, with_file_name, with_file_line { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - "Line number %d in file \"%s\" exceeds the maximum length of 2^24-1.", - file_line, file_name); - } -}; -} // namespace error - -class ByteSourceBase { -public: - virtual int read(char *buffer, int size) = 0; - virtual ~ByteSourceBase() {} -}; - -namespace detail { - -class OwningStdIOByteSourceBase : public ByteSourceBase { -public: - explicit OwningStdIOByteSourceBase(FILE *file) : file(file) { - // Tell the std library that we want to do the buffering ourself. - std::setvbuf(file, 0, _IONBF, 0); - } - - int read(char *buffer, int size) { return std::fread(buffer, 1, size, file); } - - ~OwningStdIOByteSourceBase() { std::fclose(file); } - -private: - FILE *file; -}; - -class NonOwningIStreamByteSource : public ByteSourceBase { -public: - explicit NonOwningIStreamByteSource(std::istream &in) : in(in) {} - - int read(char *buffer, int size) { - in.read(buffer, size); - return in.gcount(); - } - - ~NonOwningIStreamByteSource() {} - -private: - std::istream ∈ -}; - -class NonOwningStringByteSource : public ByteSourceBase { -public: - NonOwningStringByteSource(const char *str, long long size) - : str(str), remaining_byte_count(size) {} - - int read(char *buffer, int desired_byte_count) { - int to_copy_byte_count = desired_byte_count; - if (remaining_byte_count < to_copy_byte_count) - to_copy_byte_count = remaining_byte_count; - std::memcpy(buffer, str, to_copy_byte_count); - remaining_byte_count -= to_copy_byte_count; - str += to_copy_byte_count; - return to_copy_byte_count; - } - - ~NonOwningStringByteSource() {} - -private: - const char *str; - long long remaining_byte_count; -}; - -#ifndef CSV_IO_NO_THREAD -class AsynchronousReader { -public: - void init(std::unique_ptr arg_byte_source) { - std::unique_lock guard(lock); - byte_source = std::move(arg_byte_source); - desired_byte_count = -1; - termination_requested = false; - worker = std::thread([&] { - std::unique_lock guard(lock); - try { - for (;;) { - read_requested_condition.wait(guard, [&] { - return desired_byte_count != -1 || termination_requested; - }); - if (termination_requested) - return; - - read_byte_count = byte_source->read(buffer, desired_byte_count); - desired_byte_count = -1; - if (read_byte_count == 0) - break; - read_finished_condition.notify_one(); - } - } catch (...) { - read_error = std::current_exception(); - } - read_finished_condition.notify_one(); - }); - } - - bool is_valid() const { return byte_source != nullptr; } - - void start_read(char *arg_buffer, int arg_desired_byte_count) { - std::unique_lock guard(lock); - buffer = arg_buffer; - desired_byte_count = arg_desired_byte_count; - read_byte_count = -1; - read_requested_condition.notify_one(); - } - - int finish_read() { - std::unique_lock guard(lock); - read_finished_condition.wait( - guard, [&] { return read_byte_count != -1 || read_error; }); - if (read_error) - std::rethrow_exception(read_error); - else - return read_byte_count; - } - - ~AsynchronousReader() { - if (byte_source != nullptr) { - { - std::unique_lock guard(lock); - termination_requested = true; - } - read_requested_condition.notify_one(); - worker.join(); - } - } - -private: - std::unique_ptr byte_source; - - std::thread worker; - - bool termination_requested; - std::exception_ptr read_error; - char *buffer; - int desired_byte_count; - int read_byte_count; - - std::mutex lock; - std::condition_variable read_finished_condition; - std::condition_variable read_requested_condition; -}; -#endif - -class SynchronousReader { -public: - void init(std::unique_ptr arg_byte_source) { - byte_source = std::move(arg_byte_source); - } - - bool is_valid() const { return byte_source != nullptr; } - - void start_read(char *arg_buffer, int arg_desired_byte_count) { - buffer = arg_buffer; - desired_byte_count = arg_desired_byte_count; - } - - int finish_read() { return byte_source->read(buffer, desired_byte_count); } - -private: - std::unique_ptr byte_source; - char *buffer; - int desired_byte_count; -}; -} // namespace detail - -class LineReader { -private: - static const int block_len = 1 << 20; - std::unique_ptr buffer; // must be constructed before (and thus - // destructed after) the reader! -#ifdef CSV_IO_NO_THREAD - detail::SynchronousReader reader; -#else - detail::AsynchronousReader reader; -#endif - int data_begin; - int data_end; - - char file_name[error::max_file_name_length + 1]; - unsigned file_line; - - static std::unique_ptr open_file(const char *file_name) { - // We open the file in binary mode as it makes no difference under *nix - // and under Windows we handle \r\n newlines ourself. - FILE *file = std::fopen(file_name, "rb"); - if (file == 0) { - int x = errno; // store errno as soon as possible, doing it after - // constructor call can fail. - error::can_not_open_file err; - err.set_errno(x); - err.set_file_name(file_name); - throw err; - } - return std::unique_ptr( - new detail::OwningStdIOByteSourceBase(file)); - } - - void init(std::unique_ptr byte_source) { - file_line = 0; - - buffer = std::unique_ptr(new char[3 * block_len]); - data_begin = 0; - data_end = byte_source->read(buffer.get(), 2 * block_len); - - // Ignore UTF-8 BOM - if (data_end >= 3 && buffer[0] == '\xEF' && buffer[1] == '\xBB' && - buffer[2] == '\xBF') - data_begin = 3; - - if (data_end == 2 * block_len) { - reader.init(std::move(byte_source)); - reader.start_read(buffer.get() + 2 * block_len, block_len); - } - } - -public: - LineReader() = delete; - LineReader(const LineReader &) = delete; - LineReader &operator=(const LineReader &) = delete; - - explicit LineReader(const char *file_name) { - set_file_name(file_name); - init(open_file(file_name)); - } - - explicit LineReader(const std::string &file_name) { - set_file_name(file_name.c_str()); - init(open_file(file_name.c_str())); - } - - LineReader(const char *file_name, - std::unique_ptr byte_source) { - set_file_name(file_name); - init(std::move(byte_source)); - } - - LineReader(const std::string &file_name, - std::unique_ptr byte_source) { - set_file_name(file_name.c_str()); - init(std::move(byte_source)); - } - - LineReader(const char *file_name, const char *data_begin, - const char *data_end) { - set_file_name(file_name); - init(std::unique_ptr(new detail::NonOwningStringByteSource( - data_begin, data_end - data_begin))); - } - - LineReader(const std::string &file_name, const char *data_begin, - const char *data_end) { - set_file_name(file_name.c_str()); - init(std::unique_ptr(new detail::NonOwningStringByteSource( - data_begin, data_end - data_begin))); - } - - LineReader(const char *file_name, FILE *file) { - set_file_name(file_name); - init(std::unique_ptr( - new detail::OwningStdIOByteSourceBase(file))); - } - - LineReader(const std::string &file_name, FILE *file) { - set_file_name(file_name.c_str()); - init(std::unique_ptr( - new detail::OwningStdIOByteSourceBase(file))); - } - - LineReader(const char *file_name, std::istream &in) { - set_file_name(file_name); - init(std::unique_ptr( - new detail::NonOwningIStreamByteSource(in))); - } - - LineReader(const std::string &file_name, std::istream &in) { - set_file_name(file_name.c_str()); - init(std::unique_ptr( - new detail::NonOwningIStreamByteSource(in))); - } - - void set_file_name(const std::string &file_name) { - set_file_name(file_name.c_str()); - } - - void set_file_name(const char *file_name) { - if (file_name != nullptr) { - strncpy(this->file_name, file_name, sizeof(this->file_name)); - this->file_name[sizeof(this->file_name) - 1] = '\0'; - } else { - this->file_name[0] = '\0'; - } - } - - const char *get_truncated_file_name() const { return file_name; } - - void set_file_line(unsigned file_line) { this->file_line = file_line; } - - unsigned get_file_line() const { return file_line; } - - char *next_line() { - if (data_begin == data_end) - return nullptr; - - ++file_line; - - assert(data_begin < data_end); - assert(data_end <= block_len * 2); - - if (data_begin >= block_len) { - std::memcpy(buffer.get(), buffer.get() + block_len, block_len); - data_begin -= block_len; - data_end -= block_len; - if (reader.is_valid()) { - data_end += reader.finish_read(); - std::memcpy(buffer.get() + block_len, buffer.get() + 2 * block_len, - block_len); - reader.start_read(buffer.get() + 2 * block_len, block_len); - } - } - - int line_end = data_begin; - while (line_end != data_end && buffer[line_end] != '\n') { - ++line_end; - } - - if (line_end - data_begin + 1 > block_len) { - error::line_length_limit_exceeded err; - err.set_file_name(file_name); - err.set_file_line(file_line); - throw err; - } - - if (line_end != data_end && buffer[line_end] == '\n') { - buffer[line_end] = '\0'; - } else { - // some files are missing the newline at the end of the - // last line - ++data_end; - buffer[line_end] = '\0'; - } - - // handle windows \r\n-line breaks - if (line_end != data_begin && buffer[line_end - 1] == '\r') - buffer[line_end - 1] = '\0'; - - char *ret = buffer.get() + data_begin; - data_begin = line_end + 1; - return ret; - } -}; - -//////////////////////////////////////////////////////////////////////////// -// CSV // -//////////////////////////////////////////////////////////////////////////// - -namespace error { -const int max_column_name_length = 63; -struct with_column_name { - with_column_name() { - std::memset(column_name, 0, max_column_name_length + 1); - } - - void set_column_name(const char *column_name) { - if (column_name != nullptr) { - std::strncpy(this->column_name, column_name, max_column_name_length); - this->column_name[max_column_name_length] = '\0'; - } else { - this->column_name[0] = '\0'; - } - } - - char column_name[max_column_name_length + 1]; -}; - -const int max_column_content_length = 63; - -struct with_column_content { - with_column_content() { - std::memset(column_content, 0, max_column_content_length + 1); - } - - void set_column_content(const char *column_content) { - if (column_content != nullptr) { - std::strncpy(this->column_content, column_content, - max_column_content_length); - this->column_content[max_column_content_length] = '\0'; - } else { - this->column_content[0] = '\0'; - } - } - - char column_content[max_column_content_length + 1]; -}; - -struct extra_column_in_header : base, with_file_name, with_column_name { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - R"(Extra column "%s" in header of file "%s".)", column_name, - file_name); - } -}; - -struct missing_column_in_header : base, with_file_name, with_column_name { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - R"(Missing column "%s" in header of file "%s".)", column_name, - file_name); - } -}; - -struct duplicated_column_in_header : base, with_file_name, with_column_name { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - R"(Duplicated column "%s" in header of file "%s".)", - column_name, file_name); - } -}; - -struct header_missing : base, with_file_name { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Header missing in file \"%s\".", file_name); - } -}; - -struct too_few_columns : base, with_file_name, with_file_line { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Too few columns in line %d in file \"%s\".", file_line, - file_name); - } -}; - -struct too_many_columns : base, with_file_name, with_file_line { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Too many columns in line %d in file \"%s\".", file_line, - file_name); - } -}; - -struct escaped_string_not_closed : base, with_file_name, with_file_line { - void format_error_message() const override { - std::snprintf(error_message_buffer, sizeof(error_message_buffer), - "Escaped string was not closed in line %d in file \"%s\".", - file_line, file_name); - } -}; - -struct integer_must_be_positive : base, - with_file_name, - with_file_line, - with_column_name, - with_column_content { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - R"(The integer "%s" must be positive or 0 in column "%s" in file "%s" in line "%d".)", - column_content, column_name, file_name, file_line); - } -}; - -struct no_digit : base, - with_file_name, - with_file_line, - with_column_name, - with_column_content { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - R"(The integer "%s" contains an invalid digit in column "%s" in file "%s" in line "%d".)", - column_content, column_name, file_name, file_line); - } -}; - -struct integer_overflow : base, - with_file_name, - with_file_line, - with_column_name, - with_column_content { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - R"(The integer "%s" overflows in column "%s" in file "%s" in line "%d".)", - column_content, column_name, file_name, file_line); - } -}; - -struct integer_underflow : base, - with_file_name, - with_file_line, - with_column_name, - with_column_content { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - R"(The integer "%s" underflows in column "%s" in file "%s" in line "%d".)", - column_content, column_name, file_name, file_line); - } -}; - -struct invalid_single_character : base, - with_file_name, - with_file_line, - with_column_name, - with_column_content { - void format_error_message() const override { - std::snprintf( - error_message_buffer, sizeof(error_message_buffer), - R"(The content "%s" of column "%s" in file "%s" in line "%d" is not a single character.)", - column_content, column_name, file_name, file_line); - } -}; -} // namespace error - -using ignore_column = unsigned int; -static const ignore_column ignore_no_column = 0; -static const ignore_column ignore_extra_column = 1; -static const ignore_column ignore_missing_column = 2; - -template struct trim_chars { -private: - constexpr static bool is_trim_char(char) { return false; } - - template - constexpr static bool is_trim_char(char c, char trim_char, - OtherTrimChars... other_trim_chars) { - return c == trim_char || is_trim_char(c, other_trim_chars...); - } - -public: - static void trim(char *&str_begin, char *&str_end) { - while (str_begin != str_end && is_trim_char(*str_begin, trim_char_list...)) - ++str_begin; - while (str_begin != str_end && - is_trim_char(*(str_end - 1), trim_char_list...)) - --str_end; - *str_end = '\0'; - } -}; - -struct no_comment { - static bool is_comment(const char *) { return false; } -}; - -template struct single_line_comment { -private: - constexpr static bool is_comment_start_char(char) { return false; } - - template - constexpr static bool - is_comment_start_char(char c, char comment_start_char, - OtherCommentStartChars... other_comment_start_chars) { - return c == comment_start_char || - is_comment_start_char(c, other_comment_start_chars...); - } - -public: - static bool is_comment(const char *line) { - return is_comment_start_char(*line, comment_start_char_list...); - } -}; - -struct empty_line_comment { - static bool is_comment(const char *line) { - if (*line == '\0') - return true; - while (*line == ' ' || *line == '\t') { - ++line; - if (*line == 0) - return true; - } - return false; - } -}; - -template -struct single_and_empty_line_comment { - static bool is_comment(const char *line) { - return single_line_comment::is_comment(line) || - empty_line_comment::is_comment(line); - } -}; - -template struct no_quote_escape { - static const char *find_next_column_end(const char *col_begin) { - while (*col_begin != sep && *col_begin != '\0') - ++col_begin; - return col_begin; - } - - static void unescape(char *&, char *&) {} -}; - -template struct double_quote_escape { - static const char *find_next_column_end(const char *col_begin) { - while (*col_begin != sep && *col_begin != '\0') - if (*col_begin != quote) - ++col_begin; - else { - do { - ++col_begin; - while (*col_begin != quote) { - if (*col_begin == '\0') - throw error::escaped_string_not_closed(); - ++col_begin; - } - ++col_begin; - } while (*col_begin == quote); - } - return col_begin; - } - - static void unescape(char *&col_begin, char *&col_end) { - if (col_end - col_begin >= 2) { - if (*col_begin == quote && *(col_end - 1) == quote) { - ++col_begin; - --col_end; - char *out = col_begin; - for (char *in = col_begin; in != col_end; ++in) { - if (*in == quote && (in + 1) != col_end && *(in + 1) == quote) { - ++in; - } - *out = *in; - ++out; - } - col_end = out; - *col_end = '\0'; - } - } - } -}; - -struct throw_on_overflow { - template static void on_overflow(T &) { - throw error::integer_overflow(); - } - - template static void on_underflow(T &) { - throw error::integer_underflow(); - } -}; - -struct ignore_overflow { - template static void on_overflow(T &) {} - - template static void on_underflow(T &) {} -}; - -struct set_to_max_on_overflow { - template static void on_overflow(T &x) { - // using (std::numeric_limits::max) instead of - // std::numeric_limits::max to make code including windows.h with its max - // macro happy - x = (std::numeric_limits::max)(); - } - - template static void on_underflow(T &x) { - x = (std::numeric_limits::min)(); - } -}; - -namespace detail { -template -void chop_next_column(char *&line, char *&col_begin, char *&col_end) { - assert(line != nullptr); - - col_begin = line; - // the col_begin + (... - col_begin) removes the constness - col_end = - col_begin + (quote_policy::find_next_column_end(col_begin) - col_begin); - - if (*col_end == '\0') { - line = nullptr; - } else { - *col_end = '\0'; - line = col_end + 1; - } -} - -template -void parse_line(char *line, char **sorted_col, - const std::vector &col_order) { - for (int i : col_order) { - if (line == nullptr) - throw ::io::error::too_few_columns(); - char *col_begin, *col_end; - chop_next_column(line, col_begin, col_end); - - if (i != -1) { - trim_policy::trim(col_begin, col_end); - quote_policy::unescape(col_begin, col_end); - - sorted_col[i] = col_begin; - } - } - if (line != nullptr) - throw ::io::error::too_many_columns(); -} - -template -void parse_header_line(char *line, std::vector &col_order, - const std::string *col_name, - ignore_column ignore_policy) { - col_order.clear(); - - bool found[column_count]; - std::fill(found, found + column_count, false); - while (line) { - char *col_begin, *col_end; - chop_next_column(line, col_begin, col_end); - - trim_policy::trim(col_begin, col_end); - quote_policy::unescape(col_begin, col_end); - - for (unsigned i = 0; i < column_count; ++i) - if (col_begin == col_name[i]) { - if (found[i]) { - error::duplicated_column_in_header err; - err.set_column_name(col_begin); - throw err; - } - found[i] = true; - col_order.push_back(i); - col_begin = 0; - break; - } - if (col_begin) { - if (ignore_policy & ::io::ignore_extra_column) - col_order.push_back(-1); - else { - error::extra_column_in_header err; - err.set_column_name(col_begin); - throw err; - } - } - } - if (!(ignore_policy & ::io::ignore_missing_column)) { - for (unsigned i = 0; i < column_count; ++i) { - if (!found[i]) { - error::missing_column_in_header err; - err.set_column_name(col_name[i].c_str()); - throw err; - } - } - } -} - -template void parse(char *col, char &x) { - if (!*col) - throw error::invalid_single_character(); - x = *col; - ++col; - if (*col) - throw error::invalid_single_character(); -} - -template void parse(char *col, std::string &x) { - x = col; -} - -template void parse(char *col, const char *&x) { - x = col; -} - -template void parse(char *col, char *&x) { x = col; } - -template -void parse_unsigned_integer(const char *col, T &x) { - x = 0; - while (*col != '\0') { - if ('0' <= *col && *col <= '9') { - T y = *col - '0'; - if (x > ((std::numeric_limits::max)() - y) / 10) { - overflow_policy::on_overflow(x); - return; - } - x = 10 * x + y; - } else - throw error::no_digit(); - ++col; - } -} - -template void parse(char *col, unsigned char &x) { - parse_unsigned_integer(col, x); -} -template void parse(char *col, unsigned short &x) { - parse_unsigned_integer(col, x); -} -template void parse(char *col, unsigned int &x) { - parse_unsigned_integer(col, x); -} -template void parse(char *col, unsigned long &x) { - parse_unsigned_integer(col, x); -} -template void parse(char *col, unsigned long long &x) { - parse_unsigned_integer(col, x); -} - -template -void parse_signed_integer(const char *col, T &x) { - if (*col == '-') { - ++col; - - x = 0; - while (*col != '\0') { - if ('0' <= *col && *col <= '9') { - T y = *col - '0'; - if (x < ((std::numeric_limits::min)() + y) / 10) { - overflow_policy::on_underflow(x); - return; - } - x = 10 * x - y; - } else - throw error::no_digit(); - ++col; - } - return; - } else if (*col == '+') - ++col; - parse_unsigned_integer(col, x); -} - -template void parse(char *col, signed char &x) { - parse_signed_integer(col, x); -} -template void parse(char *col, signed short &x) { - parse_signed_integer(col, x); -} -template void parse(char *col, signed int &x) { - parse_signed_integer(col, x); -} -template void parse(char *col, signed long &x) { - parse_signed_integer(col, x); -} -template void parse(char *col, signed long long &x) { - parse_signed_integer(col, x); -} - -template void parse_float(const char *col, T &x) { - bool is_neg = false; - if (*col == '-') { - is_neg = true; - ++col; - } else if (*col == '+') - ++col; - - x = 0; - while ('0' <= *col && *col <= '9') { - int y = *col - '0'; - x *= 10; - x += y; - ++col; - } - - if (*col == '.' || *col == ',') { - ++col; - T pos = 1; - while ('0' <= *col && *col <= '9') { - pos /= 10; - int y = *col - '0'; - ++col; - x += y * pos; - } - } - - if (*col == 'e' || *col == 'E') { - ++col; - int e; - - parse_signed_integer(col, e); - - if (e != 0) { - T base; - if (e < 0) { - base = T(0.1); - e = -e; - } else { - base = T(10); - } - - while (e != 1) { - if ((e & 1) == 0) { - base = base * base; - e >>= 1; - } else { - x *= base; - --e; - } - } - x *= base; - } - } else { - if (*col != '\0') - throw error::no_digit(); - } - - if (is_neg) - x = -x; -} - -template void parse(char *col, float &x) { - parse_float(col, x); -} -template void parse(char *col, double &x) { - parse_float(col, x); -} -template void parse(char *col, long double &x) { - parse_float(col, x); -} - -template void parse(char *col, T &x) { - // Mute unused variable compiler warning - (void)col; - (void)x; - // GCC evaluates "false" when reading the template and - // "sizeof(T)!=sizeof(T)" only when instantiating it. This is why - // this strange construct is used. - static_assert(sizeof(T) != sizeof(T), - "Can not parse this type. Only builtin integrals, floats, " - "char, char*, const char* and std::string are supported"); -} - -} // namespace detail - -template , - class quote_policy = no_quote_escape<','>, - class overflow_policy = throw_on_overflow, - class comment_policy = no_comment> -class CSVReader { -private: - LineReader in; - - char *row[column_count]; - std::string column_names[column_count]; - - std::vector col_order; - - template - void set_column_names(std::string s, ColNames... cols) { - column_names[column_count - sizeof...(ColNames) - 1] = std::move(s); - set_column_names(std::forward(cols)...); - } - - void set_column_names() {} - -public: - CSVReader() = delete; - CSVReader(const CSVReader &) = delete; - CSVReader &operator=(const CSVReader &); - - template - explicit CSVReader(Args &&... args) : in(std::forward(args)...) { - std::fill(row, row + column_count, nullptr); - col_order.resize(column_count); - for (unsigned i = 0; i < column_count; ++i) - col_order[i] = i; - for (unsigned i = 1; i <= column_count; ++i) - column_names[i - 1] = "col" + std::to_string(i); - } - - char *next_line() { return in.next_line(); } - - template - void read_header(ignore_column ignore_policy, ColNames... cols) { - static_assert(sizeof...(ColNames) >= column_count, - "not enough column names specified"); - static_assert(sizeof...(ColNames) <= column_count, - "too many column names specified"); - try { - set_column_names(std::forward(cols)...); - - char *line; - do { - line = in.next_line(); - if (!line) - throw error::header_missing(); - } while (comment_policy::is_comment(line)); - - detail::parse_header_line( - line, col_order, column_names, ignore_policy); - } catch (error::with_file_name &err) { - err.set_file_name(in.get_truncated_file_name()); - throw; - } - } - - template void set_header(ColNames... cols) { - static_assert(sizeof...(ColNames) >= column_count, - "not enough column names specified"); - static_assert(sizeof...(ColNames) <= column_count, - "too many column names specified"); - set_column_names(std::forward(cols)...); - std::fill(row, row + column_count, nullptr); - col_order.resize(column_count); - for (unsigned i = 0; i < column_count; ++i) - col_order[i] = i; - } - - bool has_column(const std::string &name) const { - return col_order.end() != - std::find(col_order.begin(), col_order.end(), - std::find(std::begin(column_names), std::end(column_names), - name) - - std::begin(column_names)); - } - - void set_file_name(const std::string &file_name) { - in.set_file_name(file_name); - } - - void set_file_name(const char *file_name) { in.set_file_name(file_name); } - - const char *get_truncated_file_name() const { - return in.get_truncated_file_name(); - } - - void set_file_line(unsigned file_line) { in.set_file_line(file_line); } - - unsigned get_file_line() const { return in.get_file_line(); } - -private: - void parse_helper(std::size_t) {} - - template - void parse_helper(std::size_t r, T &t, ColType &... cols) { - if (row[r]) { - try { - try { - ::io::detail::parse(row[r], t); - } catch (error::with_column_content &err) { - err.set_column_content(row[r]); - throw; - } - } catch (error::with_column_name &err) { - err.set_column_name(column_names[r].c_str()); - throw; - } - } - parse_helper(r + 1, cols...); - } - -public: - template bool read_row(ColType &... cols) { - static_assert(sizeof...(ColType) >= column_count, - "not enough columns specified"); - static_assert(sizeof...(ColType) <= column_count, - "too many columns specified"); - try { - try { - - char *line; - do { - line = in.next_line(); - if (!line) - return false; - } while (comment_policy::is_comment(line)); - - detail::parse_line(line, row, col_order); - - parse_helper(0, cols...); - } catch (error::with_file_name &err) { - err.set_file_name(in.get_truncated_file_name()); - throw; - } - } catch (error::with_file_line &err) { - err.set_file_line(in.get_file_line()); - throw; - } - - return true; - } -}; -} // namespace io -#endif \ No newline at end of file diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h index c184d155..6bd4d6f0 100644 --- a/include/testharness/TestHarness.h +++ b/include/testharness/TestHarness.h @@ -5,6 +5,7 @@ #include "tests/Util.h" #include "testharness/ResultManager.h" #include "toolchain/ToolChain.h" +#include "Colors.h" #include #include diff --git a/include/tests/TestFile.h b/include/tests/TestFile.h index 6f14fd21..b31891ab 100644 --- a/include/tests/TestFile.h +++ b/include/tests/TestFile.h @@ -22,6 +22,9 @@ enum ParseError { RuntimeError }; +// forward declaration +class TestParser; + class TestFile { public: @@ -52,6 +55,8 @@ class TestFile { // if test has any input and if test uses input file specifically bool usesInputStream, usesInputFile, usesOut; + friend class TestParser; + protected: static uint64_t nextId; diff --git a/include/tests/TestParser.h b/include/tests/TestParser.h index de23a660..39617abd 100644 --- a/include/tests/TestParser.h +++ b/include/tests/TestParser.h @@ -43,6 +43,9 @@ class TestParser { // helper method to return the path in a FILE directive if it is good PathOrError parsePathFromLine(const std::string &line, const std::string &directive); + + // helper method to insert a newline prefixed line to a file + void insLineToFile(fs::path filePath, std::string line, bool firstInsert); // methods below look for INPUT, CHECK, INPUT_FILE, CHECK_FILE directive in a lines ParseError matchInputDirective(std::string &line); diff --git a/include/toolchain/Command.h b/include/toolchain/Command.h index c628a180..ab54d3d6 100644 --- a/include/toolchain/Command.h +++ b/include/toolchain/Command.h @@ -5,6 +5,7 @@ #include "toolchain/ExecutionState.h" #include "ExecutionState.h" +#include "Colors.h" #include #include @@ -39,10 +40,12 @@ class Command { // Ostream operator. friend std::ostream &operator<<(std::ostream&, const Command&); + // TODO: move back to private + std::string buildCommand(const ExecutionInput &input, const ExecutionOutput &output) const; + private: // Builds out best guess of the underlying command run by exec. Also adds a "redirect" as if // we were executing in the shell. In truth we're doing manual stream redirection. - std::string buildCommand(const ExecutionInput &input, const ExecutionOutput &output) const; // Resolves magic parameters to values. fs::path resolveArg(const ExecutionInput &ei, const ExecutionOutput &eo, std::string arg) const; diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp index 478efc88..be6f6fc6 100644 --- a/src/analysis/Grader.cpp +++ b/src/analysis/Grader.cpp @@ -8,15 +8,13 @@ namespace { namespace tester { -Grader::Grader(const Config &cfg) : cfg(cfg), tests() { +Grader::Grader(const Config &cfg) : cfg(cfg), tests(), results(JSON::array()) { fillModule(cfg.getTestDirPath(), tests); buildResults(); analyseResults(); } void Grader::buildResults() { - // auto &counts = analysis.addTable("counts", "Test Counts"); - // Use this loop for multiple purposes. Create the test counts, but also build up the vector of // test package names that have executables (theoretically this should be all of them). for (const auto &testPackage : tests) { @@ -34,20 +32,17 @@ void Grader::buildResults() { size_t count = 0; for (const auto &subpackage : testPackage.second) count += subpackage.second.size(); - // counts.addTestCount(name, count); } // Start running tests. Make a pass rate table for each toolchain. for (const auto &toolChain : cfg.getToolChains()) { // Table strings. std::string toolChainName = toolChain.first; - std::string tableName = toolChainName + "PassRate"; - std::string tableTitle = "Pass Rate (" + toolChainName + ")"; - // Make our table. - // auto &passRate = analysis.addTable(tableName, tableTitle); - // passRate.reserve(names); - // passRates.emplace_back(passRate); + JSON toolChainJson = { + {"toolchain", toolChain.first}, + {"results", JSON::array()} + }; // Get the toolchain and start running tests. Run over names twice since it's nxn. ToolChain tc = toolChain.second; @@ -60,29 +55,46 @@ void Grader::buildResults() { else tc.setTestedRuntime(""); - // Iterate over attackers. for (const std::string &attacker : names) { - std::cout << toolChainName << '-' << attacker << '-' << defender << ':'; + // std::cout << toolChainName << '-' << attacker << '-' << defender << ':'; + std::cout << "==== " << toolChainName << " "<< attacker << " (attacker) V.S " << defender << " (defender)"<< "\n"; // Iterate over subpackages and the contained tests from the attacker, tracking pass count. - size_t passCount = 0; + + JSON attackDefenseJson = { + {"attacker", attacker}, + {"defender", defender}, + {"testCount", 0}, + {"passCount", 0}, + {"timings", JSON::array()} + }; + + size_t passCount = 0, testCount = 0; for (const auto &subpackages : tests[attacker]) { for (const auto &test : subpackages.second) { - if (runTest(test, tc, cfg).pass) - ++passCount; - - // Status showing. Flushing every iteration isn't "ideal" but 1) I like seeing progress - // visually, 2) run time is dominated by the toolchain. Flushing doesn't hurt. - std::cout << '.'; + + TestResult result = runTest(test, tc, cfg); + + if (result.pass) { + std::cout << "."; + passCount++; + } else if (result.error) { + std::cout << "x"; + } std::cout.flush(); + testCount++; } } - std::cout << '\n'; - // Save the pass rate. - // passRate.addPassRate(defender, attacker, passCount, counts.getTestCount(attacker)); + attackDefenseJson["passCount"] = passCount; + attackDefenseJson["testCount"] = testCount; + toolChainJson["results"].push_back(attackDefenseJson); + + std::cout << '\n'; } } + + results.push_back(toolChainJson); } } @@ -98,7 +110,11 @@ void Grader::buildResults() { */ void Grader::analyseResults() { + // for (auto& attacker: names) { + // for (auto& defender: names) { + // } + // } // // Make the summary tables. // auto &totalPassRate = analysis.addTable("passSummary", "Pass Rate Summary"); // auto &totalFailRate = analysis.addTable("failSummary", "Fail Rate Summary"); diff --git a/src/main.cpp b/src/main.cpp index 61cf5547..59e3462d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,8 +20,8 @@ int main(int argc, char **argv) { // Grading means we don't run the tests like normal. Break early. if (cfg.hasGradePath()) { tester::Grader grader(cfg); - std::ofstream svFile(cfg.getGradePath()); - // grader.dump(svFile); + std::ofstream jsonOutput(cfg.getGradePath()); + grader.dump(jsonOutput); return 0; } diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index ae95f5fb..7b70f804 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -151,8 +151,9 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) TestResult result = runTest(test, toolChain, cfg); results.addResult(exeName, tcName, subPackageName, result); - std::cout << " " << test->getTestPath().stem().string() << ": " - << (result.pass ? "PASS" : "FAIL") << '\n'; + std::cout << " " << (result.pass ? (Colors::GREEN + "[PASS]" + Colors::RESET) : (Colors::RED + "[FAIL]" + Colors::RESET)) + << " " << test->getTestPath().stem().string() << '\n'; + if (result.pass) { ++packagePasses; ++subPackagePasses; diff --git a/src/tests/TestParser.cpp b/src/tests/TestParser.cpp index 78820102..dc2a12ca 100644 --- a/src/tests/TestParser.cpp +++ b/src/tests/TestParser.cpp @@ -14,6 +14,16 @@ bool fullyContains(const std::string &str, const std::string &substr) { return str.substr(pos, substr.length()) == substr; } +void TestParser::insLineToFile(fs::path filePath, std::string line, bool firstInsert) { + + std::ofstream out(filePath, std::ios::app); + + if (!firstInsert) { + out << "\n"; + } + out << line; +} + /** * @param line a single line from the test file to parse * @param directive which directive we attempt to match @@ -51,17 +61,17 @@ ParseError TestParser::matchInputDirective(std::string &line) { return ParseError::NoError; if (foundInputFile) return ParseError::DirectiveConflict; // already found an INPUT_FILE - - std::ofstream ins(testfile.getInsPath(), std::ios::app); - if (!ins.is_open()) - return ParseError::FileError; size_t findIdx = line.find(Directive::INPUT); - std::string input = line.substr(findIdx + Directive::INPUT.length()); - - ins << input << std::endl; // implicit newline - foundInput = true; + std::string inputLine = line.substr(findIdx + Directive::INPUT.length()); + try { + insLineToFile(testfile.getInsPath(), inputLine, !foundInput); + } catch (...) { + return ParseError::FileError; + } + + foundInput = true; return ParseError::NoError; } @@ -76,25 +86,16 @@ ParseError TestParser::matchCheckDirective(std::string &line) { if (foundCheckFile) return ParseError::DirectiveConflict; - std::ofstream out(testfile.getOutPath(), std::ios::app); - if (!out.is_open()) - return ParseError::FileError; - size_t findIdx = line.find(Directive::CHECK); std::string checkLine = line.substr(findIdx + Directive::CHECK.length()); - if (fs::is_empty(testfile.getOutPath())) { - if (foundCheck) { - out << "\n"; - } else { - out << checkLine; - } - } else { - out << "\n" << checkLine; + try { + insLineToFile(testfile.getOutPath(), checkLine, !foundCheck); + } catch (...) { + return ParseError::FileError; } - - foundCheck = true; - + + foundCheck = true; return ParseError::NoError; } @@ -130,10 +131,12 @@ ParseError TestParser::matchCheckFileDirective(std::string &line) { return ParseError::DirectiveConflict; PathOrError pathOrError = parsePathFromLine(line, Directive::CHECK_FILE); - if (std::holds_alternative(pathOrError)) + if (std::holds_alternative(pathOrError)) { testfile.setOutPath(std::get(pathOrError)); foundCheckFile = true; return ParseError::NoError; + } + return std::get(pathOrError); } diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp index ec95ef36..0f068f9b 100644 --- a/src/tests/TestRunning.cpp +++ b/src/tests/TestRunning.cpp @@ -12,6 +12,128 @@ // Private namespace that holds utility functions for the functions that are actually exported. You // can find the actual functions at the bottom of the file. +namespace { } // End anonymous namespace + +// void getErrorSubstr(std::string &line) { + +// std::string errorUpper = "Error:", errorLower = "error:"; + +// size_t pos1 = line.find(errorUpper); +// size_t pos2 = line.find(errorLower); + +// if (pos1 != std::string::npos) { +// line = line.substr(0, pos1 + errorUpper.length()); +// } else if (pos2 != std::string::npos) { +// line = line.substr(0, pos2 + errorLower.length()); +// } else { +// line = ""; +// } +// } + +// /** +// * @brief Tests that produce errors may contian some non-deterministic messages like PID that +// * we can not make checks for in advance. This method does a weaker partial match on the LHS of +// * the first occurence of the keyword Error: or error for bothl files. +// * +// * @returns False if there is no difference, True otherwise. Wrapped in a std::pair with diff +// * string. +// * +// * @example "MathError: line 8", "MathError: line 10" FALSE +// * @example "error: 10291 (segmentation fault)", "error: 10295 (segmentation fault)" FALSE +// * @example "TypError:", "MathError:" TRUE +// */ +// std::pair diffErrorFiles(const fs::path& file1, const fs::path& file2) { + +// std::string diffStr = ""; +// std::ifstream ifs1(file1), ifs2(file2); + +// if (!ifs1.is_open() || !ifs2.is_open()) { +// throw std::runtime_error("Failed to open file."); +// } + +// std::string errorLineOne, errorLineTwo; +// std::getline(ifs1, errorLineOne); +// std::getline(ifs2, errorLineTwo); + +// getErrorSubstr(errorLineOne); +// getErrorSubstr(errorLineTwo); + +// if (errorLineOne == errorLineTwo && (errorLineOne != "" && errorLineTwo != "")) { +// return std::make_pair(false, ""); +// } else { +// std::string diff = "+ " + errorLineOne + "\n- " + errorLineTwo; +// return std::make_pair(true, std::move(diff)); +// } +// } + + +// bool testEmittedError(const fs::path& stdoutPath) { +// std::ifstream ifs(stdoutPath); +// if (!ifs.is_open()) { +// throw std::runtime_error("Failed to open output file."); +// } + +// // std::string firstLine +// } + + + + +/* + + +TestResult runTest(const PathMatch &pm, const ToolChain &toolChain, bool quiet) { + // Try to build the test. If there's a problem running a command, then we assume failure. + ExecutionOutput eo(""); + try { + eo = toolChain.build(pm.in, pm.inStream); + } + catch (const CommandException &ce) { + if (!quiet) { + std::cout << "Command error: " << ce.what() << '\n'; + std::cout << "output " << eo.getOutputFile() << std::endl; + } + return TestResult(pm.in, false, true, ""); + } + + // Get the lines from the reference file. + std::vector expLines; + getFileLines(pm.out, expLines); + + bool isErrorTest = false; + if (expLines.size() == 1 && expLines[0].find("Error") != std::string::npos) { + isErrorTest = true; + expLines = {expLines[0]}; + } + + // Get the lines from the output file. + std::vector genLines; + + if (!isErrorTest) { // Is not an error test. + getFileLines(eo.getOutputFile(), genLines); + } + else { // Is an error test. + getFileLines(eo.getErrorFile(), genLines); + if (!genLines.empty()) + genLines = {genLines[0].substr(0, genLines[0].find(':'))}; + } + + dtl::Diff diff(expLines, genLines); + diff.compose(); + diff.composeUnifiedHunks(); + + // We failed the test. + if (!diff.getUniHunks().empty()) { + std::stringstream ss; + diff.printUnifiedFormat(ss); + return TestResult(pm.in, false, false, ss.str()); + } + + return TestResult(pm.in, true, false, ""); +} + +*/ + namespace { void dumpFile(const fs::path& filePath) { @@ -21,7 +143,7 @@ void dumpFile(const fs::path& filePath) { std::cerr << "Error opening file: " << filePath << std::endl; return; } - std::cout << ">>>" << std::endl; + std::cout << "--->" << std::endl; char ch; while (file.get(ch)) { if (ch == ' ') { @@ -32,9 +154,10 @@ void dumpFile(const fs::path& filePath) { } file.close(); - std::cout << "<<<" << std::endl; + std::cout << "<---" << std::endl; } + std::pair diffFiles(const fs::path& file1, const fs::path& file2) { std::string diffStr = ""; @@ -78,102 +201,88 @@ std::pair diffFiles(const fs::path& file1, const fs::path& fi return std::make_pair(diff, std::move(diffStr)); } +/** + * Given a file path, return the substring of the first line that conforms to the + * error testcase specification. +*/ +std::string getErrorString(const fs::path stdOutPath) { -void getErrorSubstr(std::string &line) { - - std::string errorUpper = "Error:", errorLower = "error:"; - - size_t pos1 = line.find(errorUpper); - size_t pos2 = line.find(errorLower); - - if (pos1 != std::string::npos) { - line = line.substr(0, pos1 + errorUpper.length()); - } else if (pos2 != std::string::npos) { - line = line.substr(0, pos2 + errorLower.length()); - } else { - line = ""; + std::ifstream ins(stdOutPath); // open input file stream of output file of toolchain + if (!ins.is_open()) { + throw std::runtime_error("Failed to open the generated output file of the toolchain."); } -} -/** - * @brief Tests that produce errors may contian some non-deterministic messages like PID that - * we can not make checks for in advance. This method does a weaker partial match on the LHS of - * the first occurence of the keyword Error: or error for bothl files. - * - * @returns False if there is no difference, True otherwise. Wrapped in a std::pair with diff - * string. - * - * @example "MathError: line 8", "MathError: line 10" FALSE - * @example "error: 10291 (segmentation fault)", "error: 10295 (segmentation fault)" FALSE - * @example "TypError:", "MathError:" TRUE -*/ -std::pair diffErrorFiles(const fs::path& file1, const fs::path& file2) { - - std::string diffStr = ""; - std::ifstream ifs1(file1), ifs2(file2); - if (!ifs1.is_open() || !ifs2.is_open()) { - throw std::runtime_error("Failed to open file."); + std::string firstLine; + if (!getline(ins, firstLine)) { + return std::string(""); } - std::string errorLineOne, errorLineTwo; - std::getline(ifs1, errorLineOne); - std::getline(ifs2, errorLineTwo); + if (firstLine.find("Error") != std::string::npos) { - getErrorSubstr(errorLineOne); - getErrorSubstr(errorLineTwo); - - if (errorLineOne == errorLineTwo) { - return std::make_pair(false, ""); - } else { - std::string diff = "+ " + errorLineOne + "\n- " + errorLineTwo; - return std::make_pair(true, std::move(diff)); + // expected output matches the spec, return first line + size_t colonPos = firstLine.find(":"); + if (colonPos == std::string::npos) { + // this file only contains only the LHS of the real error output. + return firstLine; + } + return firstLine.substr(0, firstLine.find(":")); } + + return std::string(""); } -} // End anonymous namespace - +} namespace tester { TestResult runTest(const std::unique_ptr &test, const ToolChain &toolChain, const Config &cfg) { - // Try to build the test. If there's a problem running a command, then we assume failure. ExecutionOutput eo(""); + + const fs::path testPath = test->getTestPath(); + const fs::path insPath = test->getInsPath(); + try { - eo = toolChain.build(test->getTestPath(), test->getInsPath()); - if (eo.getReturnValue() != 0) { - - // If the toolchain allows errors then a test with non-zero exist status may propogate here. - // If the stderr matches (partially with keywords), the expected check value, - // then the test will still pass. - if (test->usesOut) { - std::pair diff = diffErrorFiles(eo.getErrorFile(), test->getOutPath()); - if (diff.first) { - // test produces and error and stderr does not match the expected output - return TestResult(test->getTestPath(), false, true, diff.second); - } else { - // test produced an error it expected - return TestResult(test->getTestPath(), true, true, ""); - } - } else { - return TestResult(test->getTestPath(), false, true, "Test does not supply expected output."); - } - } + eo = toolChain.build(testPath, test->getInsPath()); } catch (const CommandException &ce) { + // toolchain throws errors only when allowError is false in the config if (!cfg.isQuiet()) { std::cout << "Command error: " << ce.what() << '\n'; std::cout << "output " << eo.getOutputFile() << std::endl; } - return TestResult(test->getTestPath(), false, true, ""); + return TestResult(testPath, false, true, ""); } + + + // Error test that propogated here since allowError is true in the config. + std::string genErrorString = getErrorString(eo.getErrorFile()); + std::string expErrorString = getErrorString(test->getOutPath()); + + if (eo.getReturnValue() != 0 && !genErrorString.empty() && !expErrorString.empty()) { + + if (genErrorString == expErrorString) { + return TestResult(testPath, true, true, ""); + } + + std::cout << "Exp and Gen strings don't match" << std::endl; + return TestResult(testPath, false, true, ""); + + } else { + + // Not an error test. Match the generated and expected outputs with exact diff. + std::pair diff = diffFiles(eo.getOutputFile(), test->getOutPath()); + if (diff.first) { + std::cout << "Difference between expected and generated output" << std::endl; + dumpFile(test->getTestPath()); + dumpFile(test->getOutPath()); + dumpFile(eo.getErrorFile()); + return TestResult(test->getTestPath(), false, false, ""); + } - std::pair diff = diffFiles(eo.getOutputFile(), test->getOutPath()); - if (diff.first) { - return TestResult(test->getTestPath(), false, false, diff.second); + return TestResult(test->getTestPath(), true, false, ""); } - return TestResult(test->getTestPath(), true, false, ""); } } // End namespace tester diff --git a/src/toolchain/Command.cpp b/src/toolchain/Command.cpp index fa30053f..9af5b786 100644 --- a/src/toolchain/Command.cpp +++ b/src/toolchain/Command.cpp @@ -346,6 +346,7 @@ ExecutionOutput Command::execute(const ExecutionInput &ei) const { #endif // Tell the toolchain about our output. + eo.setReturnValue(rv); return eo; } diff --git a/src/toolchain/ToolChain.cpp b/src/toolchain/ToolChain.cpp index f56aa632..1ef574a4 100644 --- a/src/toolchain/ToolChain.cpp +++ b/src/toolchain/ToolChain.cpp @@ -28,7 +28,6 @@ ExecutionOutput ToolChain::build(const fs::path &testPath, const fs::path &input int rv = eo.getReturnValue(); if (rv != 0) { - std::cout << " Command terminated with error: " << rv << std::endl; return eo; } diff --git a/tests/grade_tests/team1/001.c b/tests/grade_tests/team1/001.c index 0c20d5cb..be7958b8 100644 --- a/tests/grade_tests/team1/001.c +++ b/tests/grade_tests/team1/001.c @@ -5,11 +5,11 @@ int main() { char c[3]; - // scanf("%c", c[0]); - // scanf("%c", c[1]); - // scanf("%c", c[2]); + scanf("%c", &c[0]); + scanf("%c", &c[1]); + scanf("%c", &c[2]); - // printf("%c%c%c", c[2], c[1], c[0]); + printf("%c%c%c", c[2], c[1], c[0]); return 0; } diff --git a/tests/grade_tests/team3/002.c b/tests/grade_tests/team3/002.c deleted file mode 100644 index e9cdae16..00000000 --- a/tests/grade_tests/team3/002.c +++ /dev/null @@ -1,3 +0,0 @@ -int main() { - return 0; -} \ No newline at end of file diff --git a/tests/grades.json b/tests/grades.json new file mode 100644 index 00000000..8aacc481 --- /dev/null +++ b/tests/grades.json @@ -0,0 +1,70 @@ +[ + { + "results": [ + { + "attacker": "team1", + "defender": "team1", + "passCount": 3, + "testCount": 3, + "timings": [] + }, + { + "attacker": "team2", + "defender": "team1", + "passCount": 2, + "testCount": 2, + "timings": [] + }, + { + "attacker": "team3", + "defender": "team1", + "passCount": 1, + "testCount": 1, + "timings": [] + }, + { + "attacker": "team1", + "defender": "team2", + "passCount": 3, + "testCount": 3, + "timings": [] + }, + { + "attacker": "team2", + "defender": "team2", + "passCount": 2, + "testCount": 2, + "timings": [] + }, + { + "attacker": "team3", + "defender": "team2", + "passCount": 1, + "testCount": 1, + "timings": [] + }, + { + "attacker": "team1", + "defender": "team3", + "passCount": 3, + "testCount": 3, + "timings": [] + }, + { + "attacker": "team2", + "defender": "team3", + "passCount": 2, + "testCount": 2, + "timings": [] + }, + { + "attacker": "team3", + "defender": "team3", + "passCount": 1, + "testCount": 1, + "timings": [] + } + ], + "toolchain": "LLVM" + } +] \ No newline at end of file diff --git a/tests/run_grader_tests.sh b/tests/run_grader_tests.sh index 4dfbc0ee..0348a39b 100755 --- a/tests/run_grader_tests.sh +++ b/tests/run_grader_tests.sh @@ -23,7 +23,17 @@ fi # we need to be in the test dir to run tests cd $SCRIPT_DIR -exit 0 +echo "Tests passed." + +status=$? + +if [ $status -eq 0 ]; then + echo "Passed tests with status: $status" +else + echo "Failed tester with exit status: $status" +fi + +exit $status # TODO: Determine how to assess the output of a grading run # in CI. @@ -39,4 +49,4 @@ exit 0 # echo "Failed tester with exit status: $status" # fi -# exit $status \ No newline at end of file +# exit $status diff --git a/tests/user_tests/comments/002_comment_block.c b/tests/user_tests/comments/002_comment_block.c index de8ec618..f3b0560f 100644 --- a/tests/user_tests/comments/002_comment_block.c +++ b/tests/user_tests/comments/002_comment_block.c @@ -14,4 +14,4 @@ int main() { return 0; } -// CHECK: \ No newline at end of file +// CHECK: diff --git a/tests/user_tests/io/002.out b/tests/user_tests/io/002.out new file mode 100644 index 00000000..708caebc --- /dev/null +++ b/tests/user_tests/io/002.out @@ -0,0 +1,4 @@ +a +b +c + diff --git a/tests/user_tests/io/002_input_multi.c b/tests/user_tests/io/002_input_multi.c index 7d9e46a7..6ea3c0f2 100644 --- a/tests/user_tests/io/002_input_multi.c +++ b/tests/user_tests/io/002_input_multi.c @@ -5,24 +5,22 @@ int main(int argc, char **argv) { - char line1[25]; - char line2[25]; - - // read line 1 - if (fgets(line1, sizeof(line1), stdin) != NULL) { - printf("%s", line1); - } + char a, b; + char nl; - // read line 2 - if (fgets(line2, sizeof(line2), stdin) != NULL) { - printf("%s", line2); - } - - printf("c\n"); + scanf("%c", &a); + scanf("%c", &nl); // consume nl + scanf("%c", &b); + scanf("%c", &nl); // consume nl + + printf("%c\n", a); + printf("%c\n", b); + + printf("c"); return 0; } // CHECK:a // CHECK:b // CHECK:c -// CHECK: +// CHECK: \ No newline at end of file diff --git a/tests/user_tests/io/003_input_file.c b/tests/user_tests/io/003_input_file.c index 190a6c15..a0d6721b 100644 --- a/tests/user_tests/io/003_input_file.c +++ b/tests/user_tests/io/003_input_file.c @@ -22,4 +22,4 @@ int main() { // CHECK:15, 30, 2.71 // CHECK:22, 11, 1.41 // CHECK:8, 5, 0.99 -// CHECK: +// CHECK: \ No newline at end of file diff --git a/tests/user_tests/io/a.out b/tests/user_tests/io/a.out new file mode 100755 index 0000000000000000000000000000000000000000..8e88da2771590d9296029c77907ed00c00e8d0de GIT binary patch literal 16072 zcmeHOYit}>6~2BY4oza`?IfjHL4w*wo~#|`Rb;Y>z0SDgMdFAcL8s&0wRhEhnB7@w z7ibd#sBUU0{X<(pMT$gYQ6%yMK`NovQPYG}0wqF7K}aYCA~`5P5+F^e$a2oy^Nq*D zIzKA@RGKTTzk43vxp&UonZ4sZ^GtkaPi;+&LaS4sQ7j#A)|!|Y8aGLYAf`5}dGvRc zdO*!0x!%=jdrSjab-PqM9UDboN65Su?o`41wORo&Lq_J!)GqUcr~#E62lE=(AaW4f zWS&6Sz40PUrWSGkdU#Dlpr6N68E@4N_FNocmrhq&KxBXX~$*C2Odjn4M zJ{fPO^Kek|50vz(b=ge+#*TH_WNS8)FAuj4Z{FCtvBM}8jCOZmo>zoEyry>SKAliPlew|~SskM<|VHG2fTTTBA=_6JAx6I)G-5LBpF@yh)X7K0ZW6u9H z%SsH})<7n2XERTwlx5M0N?0YwE;?4u&g4mxDHRf1wpgWvogYv`Whargi=-x>#1mE` z{e(4OXR?ZUd_Fg-9XorscUkSmCa>4w_0~^y+l>v%>h0TQB~!)JV5a1xihaAfvW0xA z&+gCiR0eZ}yiCQ?u2miREY{H(tfx-{efXTR^e7?L?XpZ}9+An>2zFyES`1Ja^fF;B*17QZj41^g7GZ1DV z%s`j{e+K^1^uWK(Bi9j@TO3<`Qv45H^ zH?O`|sZ@@+GH=jsUcKnayivS)^)zH+kGbq_062jtFxyumNG{k3=W;$!CMAI&3w zzP`6F-Zs&8-W=G~qbmON4s~58tR%CJ4FlZ<^d{p4z8shMQ|v&Yj(OjQ$_Mdj9P`Mx0NvL-%a= zr>)i0<#ERUUa7eIBM+}%?f&e?{4dgHg8cBC7`wCHq0cGO zM_D0_$9kSVZ_s!zvo1cr`P;Q!^Jdi^qZzRxY+(k%41^g7GZ1DV%s`ldFau!*!VH8N z_-JGR^?Rt-Lroml404;R!7Cu7;z75qmZHw(BBL&FrO0?RS|Kv(15tMk`@im13hZ*S zPCKHu?_|A}QFC`m>>tx=bw=<$!4|P)T);{ zHEM-wj%f|fBUf41sHF<)UR#y=8g)1ht~=oV%xDGD3v_)PWJ)-={^>6X6C+P1`r&^u&JW5D4{llb>fb7E|S5J49m`_RF z9@00f=-B~CE@S4ki7`Jww9=@-;4S6Ii;zsPzXK}d4D zQm>J|nbt!~44xVt(k~>-aanKf|E|dNy7)H-{C_U~qp}WPmTXPDRk?0C zIV-`n>m~9`7OcT+q2JD0NvBXOS$26?B?`HrY|2R`jc9cbuEEb(cCl!WSgE{I98m*B zJD0MOxrC_c#|PiLBIG}=x|s?E1@<*bvTsaaOfzHPhWR(yB2MaA>oU)a5ES8o@o z=}ibLZi?0H-lwb`JNIngw$s|v)3ZO`XZ3B{zB6v2&Oe4?ez#gb_;P?}rLz8emhz{P zwqq;z5NQh2-yHCQPz;Oxbjd#n)9e4OWT{}K?R=8o4d~rNUdc?}Dwk48MG2!nR)47^ z+$wJ_c*ATWPkPY+nc+7Os=ih*{mlrE?k0jyfKoQdHZv_qOu-a^2H*DXom;NS=n!=oGc(E2&~BQ^-#f zEV30-S(^#c8_GJ$aHnZd*BC62$emIup$sQAOd>Bh@-DdN#z>`QiKUYi4U&$b*P3Ra zPea+c3_ab2Cl6SBl3QVE>-%@&@K308}?ITc74%j{3LyN4+H*8?{Bd$ zH|$%qq|`?0`1!&2xcL9>{)YP?bo~A3=i~bs7=fc;kpFqo-=aMb=8t_M5aVN?7@WUN zeBL91kA0+vGA_@N^9(-ra~aa{mI8e2OMz#|=5HLs-@Z%&@A1ILz7B}*D9j&zz^94N zdq&9EHv;#{V8Qug4rA2eoPv*iEO0^`F@Es)Uljg!@x#6tX!`h^vmpOf(s51@-}_#X z?-x8TNEmmn=rewgK7M{gc)%Dt`5L~qfG5c2=RYn6Ap8&lGVo-8KO_cV6huMrfO
MiQ{=Z}335YJ=q4_-g-2p{KPMZ{B(*Nwjg`M)9~j}LxhM0*1} zwCMc}@_$bjKR*`HuD~CAOx?lyf%*z{sPxpm23d#aOuTm)^oI<*L7XNZAN%o>tsHG? zy=rMn2LC@~;I%7{72ZA>@6qCYS>zEsoX6LR%f}Bso