From a9251b2234de6f1295b17c683548bf36388c3426 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sun, 11 Aug 2024 14:29:18 -0400 Subject: [PATCH 1/8] Remove docs from Tin Django Also changes links in the templates to tjcsl.github.io/tin --- tin/apps/docs/__init__.py | 0 tin/apps/docs/apps.py | 7 -- tin/apps/docs/migrations/__init__.py | 0 tin/apps/docs/urls.py | 13 --- tin/apps/docs/views.py | 49 -------- tin/settings/__init__.py | 1 - tin/templates/assignments/grader.html | 2 +- tin/templates/base.html | 2 +- tin/templates/courses/home.html | 2 +- tin/templates/docs/graders.html | 156 ------------------------- tin/templates/docs/index.html | 17 --- tin/templates/docs/sample_graders.html | 56 --------- tin/urls.py | 1 - 13 files changed, 3 insertions(+), 303 deletions(-) delete mode 100644 tin/apps/docs/__init__.py delete mode 100644 tin/apps/docs/apps.py delete mode 100644 tin/apps/docs/migrations/__init__.py delete mode 100644 tin/apps/docs/urls.py delete mode 100644 tin/apps/docs/views.py delete mode 100644 tin/templates/docs/graders.html delete mode 100644 tin/templates/docs/index.html delete mode 100644 tin/templates/docs/sample_graders.html diff --git a/tin/apps/docs/__init__.py b/tin/apps/docs/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tin/apps/docs/apps.py b/tin/apps/docs/apps.py deleted file mode 100644 index fdb0cc1a..00000000 --- a/tin/apps/docs/apps.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import annotations - -from django.apps import AppConfig - - -class DocsConfig(AppConfig): - name = "tin.apps.docs" diff --git a/tin/apps/docs/migrations/__init__.py b/tin/apps/docs/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tin/apps/docs/urls.py b/tin/apps/docs/urls.py deleted file mode 100644 index 5a2a76a2..00000000 --- a/tin/apps/docs/urls.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -from django.urls import path - -from . import views - -app_name = "docs" - -urlpatterns = [ - path("", views.index_view, name="index"), - path("graders", views.graders_view, name="graders"), - path("sample-graders", views.sample_graders_view, name="sample-graders"), -] diff --git a/tin/apps/docs/views.py b/tin/apps/docs/views.py deleted file mode 100644 index 36a2333f..00000000 --- a/tin/apps/docs/views.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -from django.shortcuts import render, reverse - -from ..auth.decorators import teacher_or_superuser_required - -# Create your views here. - - -@teacher_or_superuser_required -def index_view(request): - """The index docs page - - Args: - request: The request - """ - return render( - request, - "docs/index.html", - { - "docs_app": True, - "pages": { - reverse("docs:graders"): "Graders", - reverse("docs:sample-graders"): "Sample graders", - }, - }, - ) - - -@teacher_or_superuser_required -def graders_view(request): - """See information about how graders work - - Args: - request: The request - """ - return render(request, "docs/graders.html", {"docs_app": True, "nav_item": "Graders"}) - - -@teacher_or_superuser_required -def sample_graders_view(request): - """A sample grader - - Args: - request: The request - """ - return render( - request, "docs/sample_graders.html", {"docs_app": True, "nav_item": "Sample graders"} - ) diff --git a/tin/settings/__init__.py b/tin/settings/__init__.py index 56c9450b..c213a0cf 100644 --- a/tin/settings/__init__.py +++ b/tin/settings/__init__.py @@ -64,7 +64,6 @@ "tin.apps.courses", "tin.apps.assignments", "tin.apps.submissions", - "tin.apps.docs", "tin.apps.venvs", ] diff --git a/tin/templates/assignments/grader.html b/tin/templates/assignments/grader.html index f22586c1..60c39490 100644 --- a/tin/templates/assignments/grader.html +++ b/tin/templates/assignments/grader.html @@ -15,7 +15,7 @@

{% if assignment.is_quiz %}[QUIZ] {% endif %}{{ assignment.name }}: Grader - Please read the documentation on creating graders before you upload. + Please read the documentation on creating graders before you upload.
Size limit is 1MB.

diff --git a/tin/templates/base.html b/tin/templates/base.html index 7629a206..14f7a217 100644 --- a/tin/templates/base.html +++ b/tin/templates/base.html @@ -78,7 +78,7 @@ {% endif %} {% if docs_app %}
    • -
  • Docs
    • +
  • Docs
    • {% endif %} {% if venvs_app %}
    • diff --git a/tin/templates/courses/home.html b/tin/templates/courses/home.html index 42af6b39..b54a5207 100644 --- a/tin/templates/courses/home.html +++ b/tin/templates/courses/home.html @@ -88,7 +88,7 @@

    Courses

    New course Filter submissions Manage virtual environments - Tin documentation + Tin documentation {% endif %} diff --git a/tin/templates/docs/graders.html b/tin/templates/docs/graders.html deleted file mode 100644 index 9f948c0b..00000000 --- a/tin/templates/docs/graders.html +++ /dev/null @@ -1,156 +0,0 @@ -{% extends "base.html" %} - -{% block title %} - Turn-In: Documentation: Graders -{% endblock %} - -{% block main %} -

    Every assignment has a grader script that is run to grade each submission. This page details how to write graders - for tin.

    -

    Warning: It isn't as simple as it sounds. Certain restrictions put in place by the system create traps - that are very easy to fall into, and you should read this entire page before writing a grader script.

    - -

    When a submission is uploaded, the grader script is passed the following arguments:

    - - -

    Why is it not recommended to run the student's submission directly? Well, running student-uploaded scripts without - any kind of restrictions in place is always a bad idea, as it allows students to upload malicious scripts that, for - example, read other students' submissions and copy them to a location the student can later access. As such, a - system was put in place that limits the access that student submissions have by default. The implementation of this - system necessitated the creation of a "wrapper script" for each submission that runs the submission in an - appropriately restricted environment.

    -

    Summary: The first argument that is passed is a "wrapper script" that runs the student's submission while limiting - the access the submission has in order to prevent cheating.

    - -

    Directory structure

    -

    Currently, tin follows the following directory structure for assignments and submission.

    -

    This structure assumes a single submission from 2020awilliam on 01/01/2019 at 12:34 AM.

    - -

    Note: This is subject to change at any time, and the only lasting guarantee made about this structure - is that all of a particular student's submissions will be in the same directory. This structure is solely provided - as a visual aid.

    - -

    Restrictions placed on student submissions

    - -

    Warning: Many of these restrictions can be bypassed if the grader script uses output from a student's - submission in an unsafe way (such as eval()ing or pickle.load()ing it). tin - runs the grader script in its own restricted environment, but all this can really do is prevent the submission from - affecting code for other assignments. Please be careful.

    -

    The Internet access restriction can be lifted on a per-assignment basis by going to the assignment's "Edit" page - and checking the box labeled "Give submissions internet access." However, this should be done with caution, as - giving scripts Internet access makes it much easier for students to cheat.

    -

    The memory restriction is currently hardcoded in and cannot be changed. 1GB should be enough for many cases, but if - it becomes an issue, please contact tin's developers to - increase the limit.

    -

    Submissions can be given access to specific files by passing additional arguments to the wrapper script as follows - (these must precede any other arguments to be passed to the submission):

    - -

    If you need to upload specific read-only files for student code to use, please contact the Tin team at tin@tjhsst.edu.

    -

    The special argument -- may be passed to the wrapper script after all of the file restrictions have - been passed to denote that the wrapper should stop parsing arguments and pass the rest of the arguments directly to - the student's submission.

    - -

    Grader scripts and files

    -

    Grader scripts should only write to files in the directory containing the grader script itself (i.e. os.path.dirname(__file__)) - and the directory containing the student submission (i.e. os.path.dirname(sys.argv[2])). Files placed - in other locations may cause conflict with other graders, but these directories are guaranteed to be specific to - each grader.

    -

    The submission directories are actually created in the grader script directory. Each submission directory's name is - the username of the student, so attempting to create a file in the grader script directory with the same name as a - student's username will lead to errors.

    -

    Furthermore, files with the text "grader" in their name may be created by the server, so please avoid using these - names. For example, at the time of this writing the grader script is saved as grader.py and its log - file is saved as grader.log.

    -

    Additionally, since all of a student's submissions are placed in the same directory, files created in the - submission directory (for example, filenames passed to the submission as output files) should be given random names - to avoid conflicts in case the student uploads a second submission while their last submission has not yet been - graded.

    - -

    Examples

    -

    All of these run the submission with read-only access to input.txt in the grader script directory, - read-write access to output.txt (in practice, the names should be randomized and/or dependent on the - student's username) in the submission directory, and the command-line arguments abc and - 123.

    - 
    subprocess.run([sys.argv[1], "--read", os.path.join(os.path.dirname(__file__), "input.txt"), "--write", os.path.join(os.path.dirname(sys.argv[2]), "output.txt"), "abc", "123"])
    - (avoid this style; always pass -- so it is clear where the wrapper script arguments end) - 
    subprocess.run([sys.argv[1], "--read", os.path.join(os.path.dirname(__file__), "input.txt"), "--write", os.path.join(os.path.dirname(sys.argv[2]), "output.txt"), "--", "abc", "123"])
    - 
    subprocess.run([sys.argv[1], "--write", os.path.join(os.path.dirname(sys.argv[2]), "output.txt"), "--read", os.path.join(os.path.dirname(__file__), "input.txt"), "--", "abc", "123"])
    -

    The following code does not do what the author intended. It runs the submission with two arguments: the string - "--read" and the path to a file named input.txt in the grader script directory. It does - NOT give the script read access to input.txt.

    -

    - 

    subprocess.run([sys.argv[1], "--", "--read", os.path.join(os.path.dirname(__file__), "input.txt")])
    - -

    Grader script output

    -

    Students can only see output from the grader that has been printed on the standard output (sys.stdout). - Output on the standard error (sys.stderr) can only be viewed by teachers. This is to prevent students - from accidentally seeing a solution in the output if the grader throws an exception.

    -

    However, if the grader script exits with a non-zero status code (which Python does by default when an exception is - raised) the student will see the text [Grader error] at the end of the output. If the grader exceeds - its timeout (as set in the assignment "Edit" page), the student will see the text [Grader timed out]. - Similar text will also be added to the error output.

    -

    Automatic scoring

    -

    Each submission has a "Score" field that can be set by the grader. If this field is set, you will be able to see a - list of each student's scores on the assignment's page, which is designed to make entering grades into the gradebook - easier.

    -

    To set this field, the grader simply needs to output Score: <score> as the last line of its - output. This line is case-sensitive and the spacing must be exactly right (no trailing spaces!). Scores can be - percentages (such as 90%) or they can simply be a number of points. In either case, they are - interpreted as being out of the "Points possible" value set on the assignment "Edit" page.

    -

    Note: If the grader times out or exits with a non-zero status code, this auto-scoring will not take place. This is - to prevent inaccurate scores in the event of a grader error.

    - -{% endblock %} diff --git a/tin/templates/docs/index.html b/tin/templates/docs/index.html deleted file mode 100644 index e371ef47..00000000 --- a/tin/templates/docs/index.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "base.html" %} - -{% block title %} - Turn-In: Documentation -{% endblock %} - -{% block main %} - -

    Tin Documentation

    - - - -{% endblock %} diff --git a/tin/templates/docs/sample_graders.html b/tin/templates/docs/sample_graders.html deleted file mode 100644 index 49b1844e..00000000 --- a/tin/templates/docs/sample_graders.html +++ /dev/null @@ -1,56 +0,0 @@ -{% extends "base.html" %} - -{% block title %} - Turn-In: Documentation: Sample Graders -{% endblock %} - -{% block main %} -

    This page lists sample graders. These should be read in conjunction with the documentation on writing graders.

    - -

    A grader for a program that outputs the nth Fibonacci number

    - 
    import sys, subprocess
-
-N = 100
-cur_fib = 1
-next_fib = 1
-score = 0
-failing_cases = []
-for i in range(1, N + 1):
-    try:
-        res = subprocess.run(
-            [sys.executable, sys.argv[1], str(i)],
-            timeout=5,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-    except subprocess.TimeoutExpired:
-        print(f"Script timeout for number {i}")
-    else:
-        stdout = res.stdout.strip()
-        if not res.stderr and res.returncode == 0:
-            try:
-                if int(stdout) == cur_fib:
-                    score += 1
-                else:
-                    print(
-                        f"Invalid result for number {i} (printed {int(stdout)}, answer is {cur_fib})"
-                    )
-                    failing_cases.append(i)
-            except ValueError:
-                print(f"Non-integer printed for number {i}")
-                failing_cases.append(i)
-        else:
-            print(f"Script error for number {i}")
-            failing_cases.append(i)
-    next_fib, cur_fib = cur_fib + next_fib, next_fib
-print(f"Score: {score / N}")
-
-with open(sys.argv[4], "a") as logfile:
-    logfile.write(
-        f"User: {sys.argv[3]}; Score: {score}/{N}; Failing test cases: {', '.join(str(case) for case in failing_cases)}\n"
-    )
-
    - -{% endblock %} diff --git a/tin/urls.py b/tin/urls.py index a7934de6..20985598 100644 --- a/tin/urls.py +++ b/tin/urls.py @@ -29,7 +29,6 @@ path("assignments/", include("tin.apps.assignments.urls", namespace="assignments")), path("submissions/", include("tin.apps.submissions.urls", namespace="submissions")), path("venvs/", include("tin.apps.venvs.urls", namespace="venvs")), - path("docs/", include("tin.apps.docs.urls", namespace="docs")), path("", include("tin.apps.auth.urls", namespace="auth")), path("", include("social_django.urls", namespace="social")), ] From 19637cd4a91c0d676068fc57fd708d49de2fc477 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sun, 11 Aug 2024 14:30:02 -0400 Subject: [PATCH 2/8] Add new docs --- docs/source/contact.rst | 7 + docs/source/index.rst | 4 +- docs/source/usage.rst | 13 ++ docs/source/usage/graders/examples.rst | 22 +++ .../source/usage/graders/examples/addition.py | 96 ++++++++++ .../usage/graders/examples/addition.rst | 26 +++ .../usage/graders/examples/fibonacci.py | 66 +++++++ .../usage/graders/examples/fibonacci.rst | 31 ++++ docs/source/usage/graders/examples/file_io.py | 37 ++++ .../source/usage/graders/examples/file_io.rst | 32 ++++ docs/source/usage/graders/writing_graders.rst | 175 ++++++++++++++++++ pyproject.toml | 5 + 12 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 docs/source/contact.rst create mode 100644 docs/source/usage.rst create mode 100644 docs/source/usage/graders/examples.rst create mode 100644 docs/source/usage/graders/examples/addition.py create mode 100644 docs/source/usage/graders/examples/addition.rst create mode 100644 docs/source/usage/graders/examples/fibonacci.py create mode 100644 docs/source/usage/graders/examples/fibonacci.rst create mode 100644 docs/source/usage/graders/examples/file_io.py create mode 100644 docs/source/usage/graders/examples/file_io.rst create mode 100644 docs/source/usage/graders/writing_graders.rst diff --git a/docs/source/contact.rst b/docs/source/contact.rst new file mode 100644 index 00000000..7eb3ad8d --- /dev/null +++ b/docs/source/contact.rst @@ -0,0 +1,7 @@ +############## +Contacting Tin +############## + +If you need to get in touch with the Tin team, you can email us at tin@tjhsst.edu + +Alternatively, you can visit the Syslab at TJ to talk to us in person. diff --git a/docs/source/index.rst b/docs/source/index.rst index 7147188b..f3b466d3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,10 +18,12 @@ In order to solve this problem, Tin was invented to safely run student code subm Explore some of the technical documentation we have at our disposal! .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Contents: + usage contributing reference_index developers production + contact diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 00000000..b586a7f5 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,13 @@ +##### +Usage +##### + +If you're interested in writing a grader, check out +the pages below: + +.. toctree:: + :maxdepth: 1 + :caption: Grader Documentation + + usage/graders/writing_graders + usage/graders/examples diff --git a/docs/source/usage/graders/examples.rst b/docs/source/usage/graders/examples.rst new file mode 100644 index 00000000..8121b97e --- /dev/null +++ b/docs/source/usage/graders/examples.rst @@ -0,0 +1,22 @@ +############### +Grader Examples +############### + +If you haven't already, check out :doc:`writing_graders` before +looking at some examples. + +The following graders range from simple, to more sophisticated. + +.. toctree:: + :caption: Sample Graders + :maxdepth: 1 + + examples/file_io + examples/fibonacci + examples/addition + +To see an example of a grader that looks for a specific function name +in a student script, see :doc:`addition `. + +To see an example of a grader that gives specific permissions on files, +check out :doc:`file_io `. diff --git a/docs/source/usage/graders/examples/addition.py b/docs/source/usage/graders/examples/addition.py new file mode 100644 index 00000000..621112e0 --- /dev/null +++ b/docs/source/usage/graders/examples/addition.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import importlib.util +import sys +import traceback +from collections.abc import Callable +from pathlib import Path + +student_code_path: str = sys.argv[2] +username: str = sys.argv[3] +log_file = Path(sys.argv[4]) + +test_cases = ( + (1, 2), + (3, 4), + (1000, 20345), + (54, 78), +) + +secret_test_cases = ( + (127, 856.7), + (789.101, 101112), +) + + +def import_module(modname: str = "student_submission", func_name="add_num") -> Callable: + """Imports the student submission and returns the function with the given name.""" + spec = importlib.util.spec_from_file_location(modname, student_code_path) + + # these are probably grader errors and not student errors, so we raise an + # exception instead of printing + if spec is None: + raise ImportError(f"Could not load spec for module {student_code_path!r}") + if spec.loader is None: + raise ImportError(f"No loader found for module {student_code_path!r} with {spec=!r}") + + submission = importlib.util.module_from_spec(spec) + + if submission is None: + raise ImportError("Module spec is None") + + sys.modules[modname] = submission + + try: + spec.loader.exec_module(submission) + except Exception: + # this traceback could provide sensitive information, so we don't provide it to students + print("Could not test submission, an exception was raised while initializing.") + log_file.write_text(f"Student {username} import error:\n\n" + traceback.format_exc()) + # it's not our fault so we exit 0 + sys.exit(0) + + try: + func = getattr(submission, func_name) + except AttributeError: + print(f"Could not find function {func_name!r}") + sys.exit(0) + + return func + + +def run_submission(func: Callable) -> None: + # grade submissions + failing_cases = 0 + tol = 1e8 + for x, y in test_cases: + try: + # take into account floating point error + if func(x, y) - (x + y) > tol: + print(f"Failed on test case {x=},{y=}") + failing_cases += 1 + except Exception: + print(f"Code errored on test case {x=},{y=}") + failing_cases += 1 + + for idx, (x, y) in enumerate(secret_test_cases): + try: + if func(x, y) - (x + y) > tol: + print(f"Failed on secret test case {idx}") + failing_cases += 1 + except Exception: + print(f"Code errored on secret test case {idx}") + failing_cases += 1 + + raw = 1 - failing_cases / (len(test_cases) + len(secret_test_cases)) + # print score, rounding to two decimal places + print(f"Score: {raw * 100:.2f}%") + + +def main() -> None: + submission = import_module() + run_submission(submission) + + +if __name__ == "__main__": + main() diff --git a/docs/source/usage/graders/examples/addition.rst b/docs/source/usage/graders/examples/addition.rst new file mode 100644 index 00000000..7ffcef0f --- /dev/null +++ b/docs/source/usage/graders/examples/addition.rst @@ -0,0 +1,26 @@ +########### +Add Numbers +########### + +---------- +Assignment +---------- +Write a program that has a function called ``add_num`` that takes two parameters +:math:`x` and :math:`y`, and returns their sum :math:`x+y`. + + +---------------- +Example Solution +---------------- + +.. code-block:: python + + def add_num(x, y): + return x + y + + +-------------- +Example Grader +-------------- + +.. literalinclude:: addition.py diff --git a/docs/source/usage/graders/examples/fibonacci.py b/docs/source/usage/graders/examples/fibonacci.py new file mode 100644 index 00000000..3e7088a7 --- /dev/null +++ b/docs/source/usage/graders/examples/fibonacci.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import subprocess +import sys + +# this assignment is out of 100 points +N = 100 +score = 0 +failing_cases = [] + +# set up the fibonacci sequence so that we can check student answers +cur_fib = 1 +next_fib = 1 + +# parse information from Tin +submission, _submission_file, username, log_file, *_ = sys.argv[1:] + +for i in range(1, N + 1): + try: + # pass n as an argument to the student submission + res = subprocess.run( + [sys.executable, submission, str(i)], + # it shouldn't take more than 5 seconds + timeout=5, + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + ) + # the student submission is too slow + except subprocess.TimeoutExpired: + print(f"Script timeout for number {i}") + else: + # check if the script failed + if res.stderr or res.returncode != 0: + print(f"Script error for number {i}") + failing_cases.append(i) + continue + + try: + stdout = res.stdout.strip().decode("utf-8") + except UnicodeDecodeError: + print(f"Non-UTF-8 output for number {i}") + failing_cases.append(i) + continue + + if not stdout.isdigit(): + print(f"Non-integer printed for number {i}") + failing_cases.append(i) + continue + + student_ans = int(stdout) + if student_ans == cur_fib: + score += 1 + else: + print(f"Invalid result for number {i} (printed {student_ans}, answer is {cur_fib})") + failing_cases.append(i) + + # calculate our next fibonacci number + next_fib, cur_fib = cur_fib + next_fib, next_fib + +print(f"Score: {score / N}") + +with open(log_file, "a", encoding="utf-8") as logfile: + logfile.write( + f"User: {username}; Score: {score}/{N}; Failing test cases: {', '.join(str(case) for case in failing_cases)}\n" + ) diff --git a/docs/source/usage/graders/examples/fibonacci.rst b/docs/source/usage/graders/examples/fibonacci.rst new file mode 100644 index 00000000..f67eb818 --- /dev/null +++ b/docs/source/usage/graders/examples/fibonacci.rst @@ -0,0 +1,31 @@ +############# +Nth Fibonacci +############# + +---------- +Assignment +---------- +Write a program that takes an integer ``n`` and returns the nth Fibonacci number. + +--------------- +Sample Solution +--------------- + +.. code-block:: python + + import sys + + n = int(sys.argv[1])-1 + nums = [0, 1] + while n >= len(nums): + nums.append(nums[-1] + nums[-2]) + return nums[n] + + + + +-------------- +Example Grader +-------------- + +.. literalinclude:: fibonacci.py diff --git a/docs/source/usage/graders/examples/file_io.py b/docs/source/usage/graders/examples/file_io.py new file mode 100644 index 00000000..d1c13475 --- /dev/null +++ b/docs/source/usage/graders/examples/file_io.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +DIR = Path(__file__).parent +INPUT_FILE = DIR / "input.txt" +OUTPUT_FILE = DIR / "output.txt" + +submission = sys.argv[1] + +command = [ + sys.executable, + submission, + # give read permissions to the input + "--read", + INPUT_FILE, + # and allow them to read/write to output + "--write", + OUTPUT_FILE, + # and then pass the arguments to the student submission + "--", + INPUT_FILE, + OUTPUT_FILE, +] + +try: + resp = subprocess.run( + command, + capture_output=True, + check=True, + ) +except Exception as e: + print(f"Error in submission: {e}") +else: + print(f"Score: {100 if OUTPUT_FILE.read_text() == '2' else 0}%") diff --git a/docs/source/usage/graders/examples/file_io.rst b/docs/source/usage/graders/examples/file_io.rst new file mode 100644 index 00000000..55e15a52 --- /dev/null +++ b/docs/source/usage/graders/examples/file_io.rst @@ -0,0 +1,32 @@ +####### +File IO +####### + + +---------- +Assignment +---------- +Read from an input file and write the content to an output file. + + +--------------- +Sample Solution +--------------- + +.. code-block:: python + + import sys + + inp = sys.argv[1] + out = sys.argv[2] + with ( + open(inp, 'r') as f, + open(out, 'w') as w + ): + w.write(f.read()) + +------------- +Sample Grader +------------- + +.. literalinclude:: file_io.py diff --git a/docs/source/usage/graders/writing_graders.rst b/docs/source/usage/graders/writing_graders.rst new file mode 100644 index 00000000..14793901 --- /dev/null +++ b/docs/source/usage/graders/writing_graders.rst @@ -0,0 +1,175 @@ +################ +Writing a Grader +################ + +.. caution:: + + It isn't as simple as it sounds - there are certain traps + that are easy to fall into. Read the full page before writing a grader script. + +Tin allows you to use the full flexibility of Python (or Java) +to write a grader script. This script is responsible for evaluating +the output of a student submission, and returning a score to Tin. + +.. note:: + + In this guide, we will use Python, but the same principles apply to Java. + +---------------------- +How do I do write one? +---------------------- + +Tin passes the following arguments to the grader script: + +- The full path to the program that will run the student submission. +- The path to the student submission file - for parsing only! +- The submitting student's username. +- The path to the log file. + +You can access these in your grader script by using the :obj:`sys.argv` list +in Python. + +.. code-block:: python + + import sys + + submission, submission_file, username, log_file, *_ = sys.argv[1:] + +.. warning:: + + Do NOT use the path to the student submission file to run the student submission. + Doing so would allow students to upload malicious files, such as scripts that could read other students + submissions and copy them somewhere the student can access. + + Instead, you can run the wrapper script provided by Tin (``submission``) which will run the student + submission in a sandboxed environment, to prevent cheating. + +.. warning:: + + Do not use the ``submission_file`` to parse the student's username - the format of the + submission file path is not guaranteed to be the same in future versions of Tin. + + +Only open/write to the log file until right before the grader exits. This will minimize issues +caused by multiple submissions writing to the same file. + +You can then use this information to run the student submission (remember to use Tin's wrapper script!), +and evaluate the output of the script. + +See :doc:`examples` for examples of grader scripts. + + +----------------------------------- +Restrictions on Student Submissions +----------------------------------- + +.. attention:: + + Many of the restrictions Tin places on scripts can be bypassed if the grader script + uses student output in an unsafe way (for example, using :func:`exec` + or :func:`pickle.load`). + + +Student submissions have certain restrictions placed on them, including: + +- A 1GB memory limit +- A restriction on the amount of subprocesses that can be launched. +- Being unable to access the internet (can be configured) +- Not being able to access the submission file +- Restricted access to the rest of the filesystem. + +To allow students to access the internet, go to the assignments "Edit" page and +check the box labeled "Give submissions internet access". + +.. caution:: + + Be careful when enabling internet access - this makes it easier for + students to cheat. + +If you need to change the memory limit, please :doc:`contact Tin developers `. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Giving Students access to specific files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can give student submissions access to specific files by passing arguments +to the wrapper script: + +- ``--write ``: Give the submission read/write access to the specified file. +- ``--read ``: Give the submsission read only access to the specified file. + +Note that in both cases, ``filepath`` must be an absolute path. + +See :doc:`the file_io example ` for an example grader utilizing this feature. + +.. tip:: + + You can use the special argument ``--`` to denote the wrapper + should stop parsing arguments and pass the rest of the arguments to the submission. + For example:: + + submission --write /path/to/file -- arg1 arg2 + + will give the submission read/write access to ``/path/to/file``, and pass + ``arg1`` and ``arg2`` to the submission. + +If you need to upload specific read-only files, please :doc:`contact us `. + +------------ +Grader Files +------------ +To prevent conflicts/overwriting of other files, all graders should follow the rules below: + +Graders should only write to files in the same directory as the grader (i.e. ``Path(__file__).parent``), and the directory +containing the student submission (i.e. ``Path(sys.argv[2]).parent``). + +Do NOT create a file in the grader script directory with the same name as a students username. + +Do NOT prefix the name of any files written/read to with ``grader`` - these are reserved for the Tin server itself. + +Additionally, since all of a student's submissions are placed in the same directory, files created in the submission directory +(for example, filenames passed to the submission as output files) should be given random names to avoid +conflicts in case the student uploads a second submission while their last submission has not yet been graded. + + +------------- +Grader Output +------------- +Students can only see output from the grader that has been printed on the standard output (:obj:`sys.stdout`). +For example, students would be able to see this:: + + print("HEY YOU, STOP CHEATING!") + +However, students cannot see anything on :obj:`sys.stderr` - This is to prevent students from +seeing a solution in the output if the grader throws an exception. For example, only teachers +would be able to see the following exception:: + + raise RuntimeError("Student said 1+1=3") + +If the grader script exits with a non-zero status code (which Python does by default when an +exception is raised) the student will see the text [Grader error] at the end of the output. +If the grader exceeds its timeout (as set in the assignment "Edit" page), the student will see the text +[Grader timed out]. Similar text will also be added to the error output. + +~~~~~~~~~~~~~~~~~ +Automatic Scoring +~~~~~~~~~~~~~~~~~ +Each submission has a "Score" field that can be set by the grader. If this field is set, +you will be able to see a list of each student's scores on the assignment's page, +which is designed to make entering grades into the gradebook easier. + +To set this field, simply print ``Source: `` at the very end, to :obj:`sys.stdout`. For example:: + + print("Source: 10%") + +Note that the score can be printed as a percent (``10%``) or as a number of points. In both cases, +they are interpreted as being out of the "Points possible" value set on the assignment "Edit" page. + +.. note:: + + The autoscoring line is case sensitive and spacing must be exactly right - this means no trailing spaces are + allowed. + +.. caution:: + + If a grader exits with a non-zero status code, the auto-scoring will not take place. + This is to prevent inaccurate scores in case of a grader error. diff --git a/pyproject.toml b/pyproject.toml index c89edb08..2cda24f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,11 @@ extend-ignore-names = [ "FBT", ] +"docs/*" = [ + "BLE001", + "INP001", +] + [tool.ruff.format] docstring-code-format = true line-ending = "lf" From 7096bc8e1ae02aacec3f194d34e98ffea948bddc Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sun, 29 Sep 2024 15:02:31 -0400 Subject: [PATCH 3/8] Fix very high tolerance --- docs/source/usage/graders/examples/addition.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/usage/graders/examples/addition.py b/docs/source/usage/graders/examples/addition.py index 621112e0..03c97744 100644 --- a/docs/source/usage/graders/examples/addition.py +++ b/docs/source/usage/graders/examples/addition.py @@ -24,7 +24,13 @@ def import_module(modname: str = "student_submission", func_name="add_num") -> Callable: - """Imports the student submission and returns the function with the given name.""" + """Imports the student submission and returns the function with the given name. + + It accomplishes this by utilizing a lot of the machinery provided by the python module + ``importlib``. If you don't understand how it works, feel free to just copy paste this + function and pass a different value for the ``func_name`` parameter. + """ + spec = importlib.util.spec_from_file_location(modname, student_code_path) # these are probably grader errors and not student errors, so we raise an @@ -62,7 +68,7 @@ def import_module(modname: str = "student_submission", func_name="add_num") -> C def run_submission(func: Callable) -> None: # grade submissions failing_cases = 0 - tol = 1e8 + tol = 1e-8 for x, y in test_cases: try: # take into account floating point error From e9a1a0ee08bd44c9571fdd887f5bd925d04dcff0 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sat, 12 Oct 2024 09:42:14 -0400 Subject: [PATCH 4/8] fix typo Source -> Score --- docs/source/usage/graders/writing_graders.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/usage/graders/writing_graders.rst b/docs/source/usage/graders/writing_graders.rst index 14793901..33d81317 100644 --- a/docs/source/usage/graders/writing_graders.rst +++ b/docs/source/usage/graders/writing_graders.rst @@ -157,9 +157,9 @@ Each submission has a "Score" field that can be set by the grader. If this field you will be able to see a list of each student's scores on the assignment's page, which is designed to make entering grades into the gradebook easier. -To set this field, simply print ``Source: `` at the very end, to :obj:`sys.stdout`. For example:: +To set this field, simply print ``Score: `` at the very end, to :obj:`sys.stdout`. For example:: - print("Source: 10%") + print("Score: 10%") Note that the score can be printed as a percent (``10%``) or as a number of points. In both cases, they are interpreted as being out of the "Points possible" value set on the assignment "Edit" page. From cf1118f2894150f5b2e3cd64c757658c8a7d896d Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sat, 12 Oct 2024 09:54:46 -0400 Subject: [PATCH 5/8] Improve file-io grader to prevent race conditions --- docs/source/usage/graders/examples/file_io.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/source/usage/graders/examples/file_io.py b/docs/source/usage/graders/examples/file_io.py index d1c13475..003525cf 100644 --- a/docs/source/usage/graders/examples/file_io.py +++ b/docs/source/usage/graders/examples/file_io.py @@ -4,11 +4,16 @@ import sys from pathlib import Path -DIR = Path(__file__).parent -INPUT_FILE = DIR / "input.txt" -OUTPUT_FILE = DIR / "output.txt" - submission = sys.argv[1] +student_submission = Path(sys.argv[2]) + +# input file is in the same directory as our grader (this file) +INPUT_FILE = Path(__file__).parent / "input.txt" +# output file is in the same directory as the student submission +# This way we can avoid multiple submissions trying to write to +# the same file. +OUTPUT_FILE = student_submission.parent / "output.txt" + command = [ sys.executable, @@ -25,13 +30,14 @@ OUTPUT_FILE, ] -try: - resp = subprocess.run( - command, - capture_output=True, - check=True, - ) -except Exception as e: - print(f"Error in submission: {e}") +resp = subprocess.run( + command, + stdout=sys.stdout, + stderr=subprocess.STDOUT, + check=False, +) + +if resp.returncode == 0: + print(f"Score: {100 if OUTPUT_FILE.read_text() == INPUT_FILE.read_text() else 0}%") else: - print(f"Score: {100 if OUTPUT_FILE.read_text() == '2' else 0}%") + print("Score: 0%") From eee62814b8848e1b4509d1ac08682eddef4a68f8 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Thu, 17 Oct 2024 19:11:32 -0400 Subject: [PATCH 6/8] use absolute paths in example to match docs --- docs/source/usage/graders/examples/file_io.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/source/usage/graders/examples/file_io.py b/docs/source/usage/graders/examples/file_io.py index 003525cf..55b95f31 100644 --- a/docs/source/usage/graders/examples/file_io.py +++ b/docs/source/usage/graders/examples/file_io.py @@ -8,20 +8,24 @@ student_submission = Path(sys.argv[2]) # input file is in the same directory as our grader (this file) -INPUT_FILE = Path(__file__).parent / "input.txt" +# make sure to use the absolute path +INPUT_FILE = (Path(__file__).parent / "input.txt").resolve() + # output file is in the same directory as the student submission # This way we can avoid multiple submissions trying to write to # the same file. -OUTPUT_FILE = student_submission.parent / "output.txt" +# Again, making sure to use the absolute path +OUTPUT_FILE = (student_submission.parent / "output.txt").resolve() command = [ sys.executable, submission, # give read permissions to the input + # making sure to use the absolute path to the file "--read", INPUT_FILE, - # and allow them to read/write to output + # and allow them to read/write to the output file "--write", OUTPUT_FILE, # and then pass the arguments to the student submission @@ -37,7 +41,11 @@ check=False, ) -if resp.returncode == 0: - print(f"Score: {100 if OUTPUT_FILE.read_text() == INPUT_FILE.read_text() else 0}%") -else: +if ( + resp.returncode != 0 + or not OUTPUT_FILE.exists() + or OUTPUT_FILE.read_text() != INPUT_FILE.read_text() +): print("Score: 0%") +else: + print("Score: 100%") From a4c92d0b346b8dd584bcea6e68bdba30034c08c7 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Thu, 24 Oct 2024 09:34:41 -0400 Subject: [PATCH 7/8] Split into dev vs user docs --- docs/source/index.rst | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index f3b466d3..fe8852f7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,15 +15,24 @@ Previously, teachers in TJHSST CS classes had to manually run student code. As you can imagine, this was both time consuming, and dangerous. In order to solve this problem, Tin was invented to safely run student code submissions. -Explore some of the technical documentation we have at our disposal! +Explore some of the documentation we have at our disposal! .. toctree:: - :maxdepth: 1 - :caption: Contents: + :maxdepth: 2 + :caption: Usage Guide usage + contact + + +If you're interested in contributing a fix or a feature to Tin, +the following documents might be useful: + +.. toctree:: + :maxdepth: 1 + :caption: Development + contributing reference_index developers production - contact From c37b9073c905bcf9bc15d57203acdd815c670be7 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sat, 26 Oct 2024 19:08:27 -0400 Subject: [PATCH 8/8] fix typo --- docs/source/contributing/tests.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/contributing/tests.rst b/docs/source/contributing/tests.rst index b5802806..ae8dc6f7 100644 --- a/docs/source/contributing/tests.rst +++ b/docs/source/contributing/tests.rst @@ -183,7 +183,7 @@ For example, if you find yourself needing to create a second student often, you If a fixture only sets up something, and does not return -anything, use it with ``pytest.usefixtures``. +anything, use it with ``pytest.mark.usefixtures``. .. code-block:: python @@ -192,7 +192,7 @@ anything, use it with ``pytest.usefixtures``. assignment.is_quiz = True assignment.save() - @pytest.usefixtures("all_assigments_quiz") + @pytest.mark.usefixtures("all_assigments_quiz") def test_something(assignment): # test something, but now assignment is a quiz ...