WIP do not merge

.gitignore

@@ -84,14 +84,18 @@ nbgrader/docs/source/user_guide/managing_assignment_files_manually.rst
 nbgrader/docs/source/user_guide/managing_the_database.rst
 nbgrader/docs/source/user_guide/autograded/bitdiddle/ps1/problem1.html
 nbgrader/docs/source/user_guide/autograded/bitdiddle/ps1/problem2.html
+nbgrader/docs/source/user_guide/autograded/bitdiddle/ps1/problem3.html
 nbgrader/docs/source/user_guide/autograded/hacker/ps1/problem1.html
 nbgrader/docs/source/user_guide/autograded/hacker/ps1/problem2.html
+nbgrader/docs/source/user_guide/autograded/hacker/ps1/problem3.html
 nbgrader/docs/source/user_guide/downloaded/ps1/archive/ps1_hacker_attempt_2016-01-30-20-30-10_problem1.html
 nbgrader/docs/source/user_guide/release/ps1/problem1.html
 nbgrader/docs/source/user_guide/release/ps1/problem2.html
+nbgrader/docs/source/user_guide/release/ps1/problem3.html
 nbgrader/docs/source/user_guide/source/header.html
 nbgrader/docs/source/user_guide/source/ps1/problem1.html
 nbgrader/docs/source/user_guide/source/ps1/problem2.html
+nbgrader/docs/source/user_guide/source/ps1/problem3.html
 
 # components stuff
 node_modules

nbgrader/apps/generateassignmentapp.py

@@ -2,10 +2,11 @@
 
 import sys
 
-from traitlets import default
+from traitlets import default, Bool
+from textwrap import dedent
 
 from .baseapp import NbGrader, nbgrader_aliases, nbgrader_flags
-from ..converters import BaseConverter, GenerateAssignment, NbGraderException
+from ..converters import BaseConverter, GenerateAssignment, NbGraderException, GenerateSourceWithTests
 from traitlets.traitlets import MetaHasTraits
 from typing import List, Any
 from traitlets.config.loader import Config
@@ -51,6 +52,12 @@
         {'BaseConverter': {'force': True}},
         "Overwrite an assignment/submission if it already exists."
     ),
+    'source_with_tests': (
+        {'GenerateAssignmentApp': {'source_with_tests': True}},
+        "Generate intermediate notebooks that contain both the autogenerated test code and the solutions. "
+        "Results will be saved in the source_with_tests/ folder. "
+        "This is useful for instructors to debug problematic autogenerated test code."
+    ),
 })
 
 
@@ -62,6 +69,17 @@ class GenerateAssignmentApp(NbGrader):
     aliases = aliases
     flags = flags
 
+    source_with_tests = Bool(
+        False,
+        help=dedent(
+            """
+            Generate intermediate notebooks that contain both the autogenerated test code and the solutions.
+            Results will be saved in the source_with_tests/ folder.
+            This is useful for instructors to debug issues in autogenerated test code.
+            """
+        )
+    ).tag(config=True)
+
     examples = """
         Produce the version of the assignment that is intended to be released to
         students. This performs several modifications to the original assignment:
@@ -112,7 +130,7 @@ class GenerateAssignmentApp(NbGrader):
     @default("classes")
     def _classes_default(self) -> List[MetaHasTraits]:
         classes = super(GenerateAssignmentApp, self)._classes_default()
-        classes.extend([BaseConverter, GenerateAssignment])
+        classes.extend([BaseConverter, GenerateAssignment, GenerateSourceWithTests])
         return classes
 
     def _load_config(self, cfg: Config, **kwargs: Any) -> None:
@@ -141,6 +159,14 @@ def start(self) -> None:
         elif len(self.extra_args) == 1:
             self.coursedir.assignment_id = self.extra_args[0]
 
+
+        if self.source_with_tests:
+            converter = GenerateSourceWithTests(coursedir=self.coursedir, parent=self)
+            try:
+                converter.start()
+            except NbGraderException:
+                sys.exit(1)
+
         converter = GenerateAssignment(coursedir=self.coursedir, parent=self)
         try:
             converter.start()

nbgrader/apps/quickstartapp.py

@@ -122,6 +122,11 @@ def start(self):
         ignore_html = shutil.ignore_patterns("*.html")
         shutil.copytree(example, os.path.join(course_path, "source"), ignore=ignore_html)
 
+        # copying the autotests.yml file to the course directory
+        tests_file_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), '..', 'docs', 'source', 'user_guide', 'autotests.yml'))
+        shutil.copyfile(tests_file_path, os.path.join(course_path, 'autotests.yml'))
+
         # create the config file
         self.log.info("Generating example config file...")
         currdir = os.getcwd()

nbgrader/converters/__init__.py

@@ -5,6 +5,7 @@
 from .feedback import Feedback
 from .generate_feedback import GenerateFeedback
 from .generate_solution import GenerateSolution
+from .generate_source_with_tests import GenerateSourceWithTests
 
 __all__ = [
     "BaseConverter",
@@ -14,5 +15,6 @@
     "Autograde",
     "Feedback",
     "GenerateFeedback",
-    "GenerateSolution"
+    "GenerateSolution",
+    "GenerateSourceWithTests"
 ]

nbgrader/converters/generate_assignment.py

@@ -8,6 +8,7 @@
 from .base import BaseConverter, NbGraderException
 from ..preprocessors import (
     IncludeHeaderFooter,
+    InstantiateTests,
     ClearSolutions,
     LockCells,
     ComputeChecksums,
@@ -57,6 +58,7 @@ def _output_directory(self) -> str:
 
     preprocessors = List([
         IncludeHeaderFooter,
+        InstantiateTests,
         LockCells,
         ClearSolutions,
         ClearOutput,

nbgrader/converters/generate_source_with_tests.py

@@ -0,0 +1,49 @@
+import os
+import re
+
+from traitlets import List, default
+
+from .base import BaseConverter
+from ..preprocessors import (
+    InstantiateTests,
+    ClearOutput,
+    CheckCellMetadata
+)
+from traitlets.config.loader import Config
+from typing import Any
+from ..coursedir import CourseDirectory
+
+
+class GenerateSourceWithTests(BaseConverter):
+
+    @default("permissions")
+    def _permissions_default(self) -> int:
+        return 664 if self.coursedir.groupshared else 644
+
+    @property
+    def _input_directory(self) -> str:
+        return self.coursedir.source_directory
+
+    @property
+    def _output_directory(self) -> str:
+        return self.coursedir.source_with_tests_directory
+
+    preprocessors = List([
+        InstantiateTests,
+        ClearOutput,
+        CheckCellMetadata
+    ]).tag(config=True)
+
+    def _load_config(self, cfg: Config, **kwargs: Any) -> None:
+        super(GenerateSourceWithTests, self)._load_config(cfg, **kwargs)
+
+    def __init__(self, coursedir: CourseDirectory = None, **kwargs: Any) -> None:
+        super(GenerateSourceWithTests, self).__init__(coursedir=coursedir, **kwargs)
+
+    def start(self) -> None:
+        old_student_id = self.coursedir.student_id
+        self.coursedir.student_id = '.'
+        try:
+            super(GenerateSourceWithTests, self).start()
+        finally:
+            self.coursedir.student_id = old_student_id

nbgrader/coursedir.py

@@ -142,6 +142,18 @@ def _validate_notebook_id(self, proposal: Bunch) -> str:
         )
     ).tag(config=True)
 
+    source_with_tests_directory = Unicode(
+        'source_with_tests',
+        help=dedent(
+            """
+            The name of the directory that contains notebooks with both solutions
+            and instantiated test code (i.e., all AUTOTEST directives are removed
+            and replaced by actual test code). This corresponds to the
+            `nbgrader_step` variable in the `directory_structure` config option.
+            """
+        )
+    ).tag(config=True)
+
     submitted_directory = Unicode(
         'submitted',
         help=dedent(