diff --git a/checker/__main__.py b/checker/__main__.py index 4abad79..b990515 100644 --- a/checker/__main__.py +++ b/checker/__main__.py @@ -94,7 +94,8 @@ def check( deadlines_config=private_course_driver.get_deadlines_file_path(), ) tester = Tester.create( - system=course_config.system, + root=root, + course_config=course_config, cleanup=not no_clean, dry_run=dry_run, ) @@ -157,7 +158,8 @@ def grade( deadlines_config=private_course_driver.get_deadlines_file_path(), ) tester = Tester.create( - system=course_config.system, + root=execution_folder, + course_config=course_config, ) grade_on_ci( diff --git a/checker/course/config.py b/checker/course/config.py index 0564a18..ef192f6 100644 --- a/checker/course/config.py +++ b/checker/course/config.py @@ -48,6 +48,7 @@ class CourseConfig: # checker default layout: str = 'groups' executor: str = 'sandbox' + tester_path: str | None = None # info links: dict[str, str] | None = None diff --git a/checker/testers/tester.py b/checker/testers/tester.py index c83319a..8957589 100644 --- a/checker/testers/tester.py +++ b/checker/testers/tester.py @@ -7,11 +7,25 @@ from pathlib import Path from typing import Any +from ..course import CourseConfig from ..exceptions import RunFailedError, TaskTesterTestConfigException, TesterNotImplemented from ..executors.sandbox import Sandbox from ..utils.print import print_info +def _create_external_tester(tester_path: Path, dry_run: bool, cleanup: bool): + globls = {} + with open(tester_path) as f: + tester_code = compile(f.read(), tester_path.absolute(), 'exec') + exec(tester_code, globls) + tester_cls = globls.get('CustomTester') + if tester_cls is None: + raise TesterNotImplemented(f'class CustomTester not found in file {tester_path}') + if not issubclass(tester_cls, Tester): + raise TesterNotImplemented(f'class CustomTester in {tester_path} is not inherited from testers.Tester') + return tester_cls(dry_run=dry_run, cleanup=cleanup) + + class Tester: """Entrypoint to testing system Tester holds the course object and manage testing of single tasks, @@ -27,7 +41,6 @@ class TaskTestConfig: """Task Tests Config Configure how task will copy files, check, execute and so on """ - pass @classmethod def from_json( @@ -75,7 +88,8 @@ def __init__( @classmethod def create( cls, - system: str, + root: Path, + course_config: CourseConfig, cleanup: bool = True, dry_run: bool = False, ) -> 'Tester': @@ -87,6 +101,7 @@ def create( @param dry_run: Setup dry run mode (really executes nothing) @return: Configured Tester object (python, cpp, etc.) """ + system = course_config.system if system == 'python': from . import python return python.PythonTester(cleanup=cleanup, dry_run=dry_run) @@ -96,6 +111,11 @@ def create( elif system == 'cpp': from . import cpp return cpp.CppTester(cleanup=cleanup, dry_run=dry_run) + elif system == 'external': + path = course_config.tester_path + if path is None: + raise TesterNotImplemented(f'tester_path is not specified in course config') + return _create_external_tester(root / path, cleanup=cleanup, dry_run=dry_run) else: raise TesterNotImplemented(f'Tester for <{system}> are not supported right now') diff --git a/tests/testers/test_tester.py b/tests/testers/test_tester.py index 2d2d3b2..b004e2b 100644 --- a/tests/testers/test_tester.py +++ b/tests/testers/test_tester.py @@ -10,6 +10,27 @@ from checker.testers.make import MakeTester from checker.testers.python import PythonTester from checker.testers.tester import Tester +from checker.course import CourseConfig + + +def create_test_course_config(**kwargs) -> CourseConfig: + return CourseConfig( + name='test', + deadlines='', + templates='', + manytask_url='', + course_group='', + public_repo='', + students_group='', + **kwargs, + ) + +def write_tester_to_file(path: Path, content: str) -> Path: + filename = path / 'tester.py' + content = inspect.cleandoc(content) + with open(filename, 'w') as f: + f.write(content) + return filename class TestTester: @@ -19,12 +40,45 @@ class TestTester: ('make', MakeTester), ]) def test_right_tester_created(self, tester_name: str, tester_class: Type[Tester]) -> None: - tester = Tester.create(tester_name) + course_config = create_test_course_config(system=tester_name) + tester = Tester.create(root=Path(), course_config=course_config) assert isinstance(tester, tester_class) + def test_external_tester(self, tmp_path: Path): + TESTER = """ + from checker.testers import Tester + class CustomTester(Tester): + definitely_external_tester = 'Yes!' + """ + course_config = create_test_course_config(system='external', tester_path='tester.py') + write_tester_to_file(tmp_path, TESTER) + tester = Tester.create(root=tmp_path, course_config=course_config) + assert hasattr(tester, 'definitely_external_tester') + + NOT_A_TESTER = """ + class NotATester: + definitely_external_tester = 'Yes!' + """ + + NOT_INHERITED_TESTER = """ + class CustomTester: + definitely_external_tester = 'Yes!' + """ + + @pytest.mark.parametrize('tester_content', [ + NOT_A_TESTER, + NOT_INHERITED_TESTER, + ]) + def test_invalid_external_tester(self, tmp_path: Path, tester_content): + course_config = create_test_course_config(system='external', tester_path='tester.py') + write_tester_to_file(tmp_path, tester_content) + with pytest.raises(TesterNotImplemented): + Tester.create(root=tmp_path, course_config=course_config) + def test_wrong_tester(self) -> None: + course_config = create_test_course_config(system='definitely-wrong-tester') with pytest.raises(TesterNotImplemented): - Tester.create('definitely-wrong-tester') + Tester.create(root=Path(), course_config=course_config) @dataclass