From c285cd00a9aabcb778418238c24c574bf956d4f6 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Tue, 11 Aug 2020 09:55:49 +0300 Subject: [PATCH 1/7] added typing --- pycrunch/session/combined_coverage.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pycrunch/session/combined_coverage.py b/pycrunch/session/combined_coverage.py index 0ad9c95..29b7df9 100644 --- a/pycrunch/session/combined_coverage.py +++ b/pycrunch/session/combined_coverage.py @@ -1,4 +1,5 @@ from collections import defaultdict +from typing import Dict, Any, Union, Set from pycrunch.api.shared import file_watcher from pycrunch.session import config @@ -11,6 +12,9 @@ def __init__(self, filename, lines_covered): class FileStatistics: + filename: str + lines_with_entrypoints: Dict[str, Set[str]] + def __init__(self, filename): self.filename = filename # line by line, each line contains one or multiple tests @@ -35,17 +39,22 @@ def clear_file_from_test(self, fqn): class CombinedCoverage: + # FQN -> Files touched during run - i.e.: [file1.py, file2.py] + file_dependencies_by_tests: Dict[str, Set[str]] + # filename.py -> set(fqn, fqn) + dependencies: Dict[str, Set[str]] + + # filename -> FileStatistics + files: Dict[str, FileStatistics] """ files[] -> line 1 -> [test1, test2] line 2 -> [test2] """ def __init__(self): - # filename -> FileStatistics self.files = dict() # all files involved in execution of test. # FQN will end up showing in multiple files if dependent file was used during run - # filename.py -> set(fqn, fqn) self.dependencies = defaultdict(set) self.file_dependencies_by_tests = defaultdict(set) # in format @@ -56,11 +65,11 @@ def __init__(self): # self.aggregated_results = defaultdict(dict) pass - def mark_dependency(self, filename, fqn): + def mark_dependency(self, filename: str, fqn: str): if fqn not in self.dependencies[filename]: self.dependencies[filename].add(fqn) - def test_did_removed(self, fqn): + def test_did_removed(self, fqn: str): # clear dependencies for filename in self.dependencies: self.dependencies[filename].discard(fqn) From d20982651caa1db7fb85204a0cea32f10f541c06 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Tue, 11 Aug 2020 10:26:11 +0300 Subject: [PATCH 2/7] v1.2 run tests under debug --- pycrunch/api/socket_handlers.py | 12 ++++++---- pycrunch/child_runtime/child_config.py | 16 ++++++++++++++ pycrunch/child_runtime/client_protocol.py | 5 +++-- pycrunch/child_runtime/test_runner.py | 11 +++++++--- .../crossprocess/multiprocess_test_runner.py | 8 +++++-- pycrunch/multiprocess_child_main.py | 11 ++++++---- pycrunch/pipeline/file_modification_task.py | 4 ++-- pycrunch/pipeline/run_test_task.py | 22 +++++++++++++++---- .../pytest_support/pytest_runner_engine.py | 12 ++++++---- 9 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 pycrunch/child_runtime/child_config.py diff --git a/pycrunch/api/socket_handlers.py b/pycrunch/api/socket_handlers.py index 2fb166a..c5511c1 100644 --- a/pycrunch/api/socket_handlers.py +++ b/pycrunch/api/socket_handlers.py @@ -10,7 +10,7 @@ from pycrunch.api.shared import pipe from pycrunch.pipeline import execution_pipeline from pycrunch.pipeline.download_file_task import DownloadFileTask -from pycrunch.pipeline.run_test_task import RunTestTask +from pycrunch.pipeline.run_test_task import RunTestTask, RemoteDebugParams from pycrunch.runner.pipeline_dispatcher import dispather_thread from pycrunch.session import config from pycrunch.session.state import engine @@ -51,7 +51,7 @@ async def handle_my_custom_event(sid, json): action = json.get('action') if action == 'discovery': await engine.will_start_test_discovery() - if action == 'run-tests': + if action == 'run-tests' or action == 'debug-tests': if 'tests' not in json: logger.error('run-tests command received, but no tests specified') return @@ -62,8 +62,12 @@ async def handle_my_custom_event(sid, json): fqns.add(test['fqn']) tests_to_run = all_tests.collect_by_fqn(fqns) + debug_params = RemoteDebugParams.disabled() + if action == 'debug-tests': + debugger_port = json.get('debugger_port') + debug_params = RemoteDebugParams(True, debugger_port) - execution_pipeline.add_task(RunTestTask(tests_to_run)) + execution_pipeline.add_task(RunTestTask(tests_to_run, debug_params)) if action == 'load-file': filename = json.get('filename') logger.debug('download_file ' + filename) @@ -106,7 +110,7 @@ async def connect(sid, environ): engine_mode=engine.get_engine_mode(), version=dict( major=1, - minor=1, + minor=2, ) ) ) diff --git a/pycrunch/child_runtime/child_config.py b/pycrunch/child_runtime/child_config.py new file mode 100644 index 0000000..1f87956 --- /dev/null +++ b/pycrunch/child_runtime/child_config.py @@ -0,0 +1,16 @@ +class ChildRuntimeConfig: + def __init__(self): + self.load_pytest_plugins = False + self.enable_remote_debug = False + self.remote_debug_port = -1 + self.runtime_engine = 'pytest' + + def use_engine(self, new_engine): + self.runtime_engine = new_engine + + def enable_remote_debugging(self, port_to_use): + self.enable_remote_debug = True + self.remote_debug_port = int(port_to_use) + + +child_config = ChildRuntimeConfig() diff --git a/pycrunch/child_runtime/client_protocol.py b/pycrunch/child_runtime/client_protocol.py index 94fd385..cdd4693 100644 --- a/pycrunch/child_runtime/client_protocol.py +++ b/pycrunch/child_runtime/client_protocol.py @@ -57,10 +57,11 @@ def data_received(self, data): if self.engine_to_use == 'django': config.prepare_django() - runner_engine = PyTestRunnerEngine(config.load_pytest_plugins) + from pycrunch.child_runtime.child_config import child_config + runner_engine = PyTestRunnerEngine(child_config) # should have env from pycrunch config heve - r = TestRunner(runner_engine, timeline) + r = TestRunner(runner_engine, timeline, child_config) timeline.mark_event(f'Run: about to run tests') try: timeline.mark_event(f'Run: total tests planned: {len(msg.task.tests)}') diff --git a/pycrunch/child_runtime/test_runner.py b/pycrunch/child_runtime/test_runner.py index d3b514f..f17495f 100644 --- a/pycrunch/child_runtime/test_runner.py +++ b/pycrunch/child_runtime/test_runner.py @@ -1,4 +1,5 @@ from pycrunch.api.serializers import CoverageRun +from pycrunch.child_runtime.child_config import ChildRuntimeConfig from pycrunch.child_runtime.coverage_hal import CoverageAbstraction from pycrunch.insights.variables_inspection import InsightTimeline, inject_timeline from pycrunch.introspection.clock import clock @@ -7,9 +8,10 @@ DISABLE_COVERAGE = False class TestRunner: - def __init__(self, runner_engine, timeline): - self.timeline = timeline + def __init__(self, runner_engine, timeline, child_config): self.runner_engine = runner_engine + self.timeline = timeline + self.child_config = child_config def run(self, tests): self.timeline.mark_event('Run: inside run method') @@ -34,7 +36,10 @@ def run(self, tests): # --- # checked, there are 2x improvement for small files (0.06 vs 0.10, but still # slow as before on 500+ tests in one file - cov = CoverageAbstraction(DISABLE_COVERAGE, self.timeline) + should_disable_coverage = DISABLE_COVERAGE + if self.child_config.enable_remote_debug: + should_disable_coverage = True + cov = CoverageAbstraction(should_disable_coverage, self.timeline) cov.start() with capture_stdout() as get_value: diff --git a/pycrunch/crossprocess/multiprocess_test_runner.py b/pycrunch/crossprocess/multiprocess_test_runner.py index 41375b4..4ab91be 100644 --- a/pycrunch/crossprocess/multiprocess_test_runner.py +++ b/pycrunch/crossprocess/multiprocess_test_runner.py @@ -13,13 +13,14 @@ class MultiprocessTestRunner: - def __init__(self, timeout: Optional[float], timeline, test_run_scheduler): + def __init__(self, timeout: Optional[float], timeline, test_run_scheduler, remote_debug_params: "RemoteDebugParams"): self.client_connections: List[TestRunnerServerProtocol] = [] self.completion_futures = [] self.timeline = timeline self.timeout = timeout self.results = None self.test_run_scheduler = test_run_scheduler + self.remote_debug_params = remote_debug_params def results_did_become_available(self, results): logger.debug('results avail:') @@ -102,7 +103,10 @@ async def run(self, tests): def get_command_line_for_child(self, port, task_id): engine_root = f' {config.engine_directory}{os.sep}pycrunch{os.sep}multiprocess_child_main.py ' hardcoded_path = engine_root + f'--engine={config.runtime_engine} --port={port} --task-id={task_id} --load-pytest-plugins={str(config.load_pytest_plugins).lower()}' - return sys.executable + hardcoded_path + remote_debug_str = '' + if self.remote_debug_params.enabled: + remote_debug_str = f' --enable-remote-debug --remote-debugger-port={self.remote_debug_params.port}' + return sys.executable + hardcoded_path + f'{remote_debug_str}' def create_server_protocol(self): loop = asyncio.get_event_loop() diff --git a/pycrunch/multiprocess_child_main.py b/pycrunch/multiprocess_child_main.py index b353fb3..cc8b0c0 100644 --- a/pycrunch/multiprocess_child_main.py +++ b/pycrunch/multiprocess_child_main.py @@ -53,15 +53,18 @@ async def main(): parser.add_argument('--port', help='PyCrunch-Engine server port to connect') parser.add_argument('--task-id', help='Id of task when multiple test runners ran at same time') parser.add_argument('--load-pytest-plugins', help='If this is true, execution will be slower.') - + parser.add_argument('--enable-remote-debug', action='store_true', help='If this is true, remote debug will be enabled on a --remote-debugger-port') + parser.add_argument('--remote-debugger-port', help='If remote debug is enabled, this will specify a port used to connect to PyCharm pudb') args = parser.parse_args() timeline.mark_event('ArgumentParser: parse_args completed') engine_to_use = args.engine if engine_to_use: - from pycrunch.session import config - config.runtime_engine_will_change(engine_to_use) + from pycrunch.child_runtime.child_config import child_config + child_config.use_engine(engine_to_use) if args.load_pytest_plugins.lower() == 'true': - config.load_pytest_plugins = True + child_config.load_pytest_plugins = True + if args.enable_remote_debug: + child_config.enable_remote_debugging(args.remote_debugger_port) timeline.mark_event('Before run') diff --git a/pycrunch/pipeline/file_modification_task.py b/pycrunch/pipeline/file_modification_task.py index b6c34e6..3f6601c 100644 --- a/pycrunch/pipeline/file_modification_task.py +++ b/pycrunch/pipeline/file_modification_task.py @@ -4,7 +4,7 @@ from pycrunch.discovery.simple import SimpleTestDiscovery from pycrunch.pipeline import execution_pipeline from pycrunch.pipeline.abstract_task import AbstractTask -from pycrunch.pipeline.run_test_task import RunTestTask +from pycrunch.pipeline.run_test_task import RunTestTask, RemoteDebugParams from pycrunch.session import state from pycrunch.session.combined_coverage import combined_coverage from pycrunch.session.file_map import test_map @@ -65,7 +65,7 @@ async def run(self): tests_to_run = state.engine.all_tests.collect_by_fqn(execution_plan) dirty_tests = self.consider_engine_mode(tests_to_run) - execution_pipeline.add_task(RunTestTask(dirty_tests)) + execution_pipeline.add_task(RunTestTask(dirty_tests, RemoteDebugParams.disabled())) pass; diff --git a/pycrunch/pipeline/run_test_task.py b/pycrunch/pipeline/run_test_task.py index d268a31..2e0d935 100644 --- a/pycrunch/pipeline/run_test_task.py +++ b/pycrunch/pipeline/run_test_task.py @@ -1,7 +1,6 @@ import asyncio import os -from pprint import pprint -from typing import Dict, Any +from typing import Dict, Any, Optional from pycrunch.api import shared from pycrunch.api.serializers import CoverageRun @@ -42,8 +41,18 @@ def is_failed(self): return self.status != 'success' +class RemoteDebugParams: + def __init__(self, enabled: bool, port: Optional[int] = None): + self.port = port + self.enabled = enabled + + @classmethod + def disabled(cls): + return RemoteDebugParams(False) + class RunTestTask(AbstractTask): - def __init__(self, tests): + def __init__(self, tests, remote_debug_params: RemoteDebugParams): + self.remote_debug_params = remote_debug_params self.timestamp = shared.timestamp() self.tests = tests self.results = None @@ -138,6 +147,10 @@ def convert_result_to_json(self, run_results): return results_as_json def post_process_combined_coverage(self, run_results): + if self.remote_debug_params.enabled: + self.timeline.mark_event('Postprocessing: combined coverage will not be recomputed.') + return + self.timeline.mark_event('Postprocessing: combined coverage, line hits, dependency tree') combined_coverage.add_multiple_results(run_results) self.timeline.mark_event('Postprocessing: completed') @@ -151,7 +164,8 @@ def create_test_runner(self): test_run_scheduler=TestRunScheduler( cpu_cores=config.cpu_cores, threshold=config.multiprocessing_threshold - ) + ), + remote_debug_params=self.remote_debug_params, ) return runner diff --git a/pycrunch/plugins/pytest_support/pytest_runner_engine.py b/pycrunch/plugins/pytest_support/pytest_runner_engine.py index 4382e78..3cde5cc 100644 --- a/pycrunch/plugins/pytest_support/pytest_runner_engine.py +++ b/pycrunch/plugins/pytest_support/pytest_runner_engine.py @@ -13,12 +13,12 @@ class PyTestRunnerEngine(_abstract_runner.Runner): - def __init__(self, load_plugins): + def __init__(self, child_config): """ - :type load_plugins: bool + :type child_config: pycrunch.child_runtime.child_config.ChildRuntimeConfig """ - self.load_plugins = load_plugins + self.child_config = child_config def run_test(self, test): @@ -39,7 +39,7 @@ def run_test(self, test): additional_pytest_args = ['-qs' ] plugins_arg = [] - if not self.load_plugins: + if not self.child_config.load_pytest_plugins: os.environ['PYTEST_DISABLE_PLUGIN_AUTOLOAD'] = 'True' plugins_arg += ['-p', 'no:junitxml'] @@ -47,6 +47,10 @@ def run_test(self, test): # , '-p', 'no:helpconfig', - cannot be disabled all_args = additional_pytest_args + plugins_arg # print(all_args, file=sys.__stdout__) + if self.child_config.enable_remote_debug: + import pydevd_pycharm + pydevd_pycharm.settrace('127.0.0.1', suspend=False, port=self.child_config.remote_debug_port, stdoutToServer=True, stderrToServer=True) + pytest.main([fqn_test_to_run] + all_args, plugins=[plugin]) # pytest.main([fqn_test_to_run, '-qs'], plugins=[plugin]) From 94aa440096a486443cf699e3c46cdffc1f58e3d4 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Tue, 11 Aug 2020 13:31:46 +0300 Subject: [PATCH 3/7] Execution timeout will not be applied during test debugging --- pycrunch/api/socket_handlers.py | 3 ++- pycrunch/crossprocess/multiprocess_test_runner.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pycrunch/api/socket_handlers.py b/pycrunch/api/socket_handlers.py index c5511c1..f60f6e7 100644 --- a/pycrunch/api/socket_handlers.py +++ b/pycrunch/api/socket_handlers.py @@ -62,10 +62,11 @@ async def handle_my_custom_event(sid, json): fqns.add(test['fqn']) tests_to_run = all_tests.collect_by_fqn(fqns) - debug_params = RemoteDebugParams.disabled() if action == 'debug-tests': debugger_port = json.get('debugger_port') debug_params = RemoteDebugParams(True, debugger_port) + else: + debug_params = RemoteDebugParams.disabled() execution_pipeline.add_task(RunTestTask(tests_to_run, debug_params)) if action == 'load-file': diff --git a/pycrunch/crossprocess/multiprocess_test_runner.py b/pycrunch/crossprocess/multiprocess_test_runner.py index 4ab91be..e33eced 100644 --- a/pycrunch/crossprocess/multiprocess_test_runner.py +++ b/pycrunch/crossprocess/multiprocess_test_runner.py @@ -60,11 +60,11 @@ async def run(self, tests): try: await asyncio.wait_for( asyncio.gather(*child_waiters), - timeout=self.timeout + timeout=self.timeout_if_non_debug() ) except (asyncio.TimeoutError, asyncio.CancelledError) as e: timeout_reached = True - logger.warning(f'Reached execution timeout of {self.timeout} seconds. ') + logger.warning(f'Reached execution timeout of {self.timeout_if_non_debug()} seconds. ') for _ in subprocesses_results: try: _.kill() @@ -77,7 +77,7 @@ async def run(self, tests): try: demo_results = await asyncio.wait_for( asyncio.gather(*self.completion_futures, return_exceptions=True), - timeout=self.timeout + timeout=self.timeout_if_non_debug() ) except asyncio.TimeoutError as ex: print(ex) @@ -100,6 +100,11 @@ async def run(self, tests): raise asyncio.TimeoutError('Test execution timeout.') return _results + def timeout_if_non_debug(self) -> Optional[float]: + if self.remote_debug_params.enabled: + return None + return self.timeout + def get_command_line_for_child(self, port, task_id): engine_root = f' {config.engine_directory}{os.sep}pycrunch{os.sep}multiprocess_child_main.py ' hardcoded_path = engine_root + f'--engine={config.runtime_engine} --port={port} --task-id={task_id} --load-pytest-plugins={str(config.load_pytest_plugins).lower()}' From dcdc09a6953b493ba4aa443976dd0582f93e3a69 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Thu, 13 Aug 2020 11:31:05 +0300 Subject: [PATCH 4/7] Added runtime deadline for process without connections --- pycrunch/api/socket_handlers.py | 6 ++- pycrunch/child_runtime/client_protocol.py | 2 +- pycrunch/main.py | 28 +---------- pycrunch/watchdog/connection_watchdog.py | 57 +++++++++++++++++++++++ 4 files changed, 64 insertions(+), 29 deletions(-) create mode 100644 pycrunch/watchdog/connection_watchdog.py diff --git a/pycrunch/api/socket_handlers.py b/pycrunch/api/socket_handlers.py index f60f6e7..f221680 100644 --- a/pycrunch/api/socket_handlers.py +++ b/pycrunch/api/socket_handlers.py @@ -16,6 +16,7 @@ from pycrunch.session.state import engine from pycrunch.shared.models import all_tests from . import shared +from ..watchdog.connection_watchdog import connection_watchdog from ..watchdog.tasks import TerminateTestExecutionTask from ..watchdog.watchdog import watchdog_dispather_thread from ..watchdog.watchdog_pipeline import watchdog_pipeline @@ -103,7 +104,7 @@ async def connect(sid, environ): global thread global watchdog_thread logger.debug('Client test_connected') - + connection_watchdog.connection_established() await pipe.push( event_type='connected', **dict( @@ -128,4 +129,5 @@ async def connect(sid, environ): @shared.sio.event def disconnect(sid): - logger.debug('Client disconnected') \ No newline at end of file + logger.debug('Client disconnected') + connection_watchdog.connection_lost() diff --git a/pycrunch/child_runtime/client_protocol.py b/pycrunch/child_runtime/client_protocol.py index cdd4693..a512470 100644 --- a/pycrunch/child_runtime/client_protocol.py +++ b/pycrunch/child_runtime/client_protocol.py @@ -119,7 +119,7 @@ def mark_all_done(self): def connection_lost(self, exc): self.timeline.mark_event(f'TCP: Connection to server lost') - print(f'[{os.getpid()}] [task_id: {self.task_id}] - The connection to server lost') + print(f'[{os.getpid()}] [task_id: {self.task_id}] - The connection to parent pycrunch-engine process lost') self.on_con_lost.set_result(True) def error_received(self, exc): diff --git a/pycrunch/main.py b/pycrunch/main.py index 958fdd6..564fd54 100644 --- a/pycrunch/main.py +++ b/pycrunch/main.py @@ -9,7 +9,7 @@ from pycrunch import web_ui from pycrunch.session import config - +from pycrunch.watchdog.connection_watchdog import connection_watchdog package_directory = Path(__file__).parent print(package_directory) @@ -49,31 +49,6 @@ def run(): from pycrunch.api import shared import pycrunch.api.socket_handlers - # sio = socketio.Server() - - - # settings = { - # "static_path": os.path.join(os.path.dirname(__file__), "front/dist"), - # - # } - - # _Handler = socketio.get_tornado_handler(sio) - # - # class SocketHandler(_Handler): - # def check_origin(self, origin): - # return True - # - # - # app = tornado.web.Application( - # [ - # (r'/ui/(.*)', tornado.web.StaticFileHandler, {"path" : 'front/dist'}), - # (r'/js/(.*)', tornado.web.StaticFileHandler, {"path" : 'front/dist/js'}), - # (r'/css/(.*)', tornado.web.StaticFileHandler, {"path" : 'front/dist/css'}), - # (r"/socket.io/", SocketHandler), - # ], - # # **settings - # # ... other application options - # ) app = web.Application() sio.attach(app) @@ -83,6 +58,7 @@ def run(): loop = asyncio.get_event_loop() + task = loop.create_task(connection_watchdog.watch_client_connection_loop()) loop.set_debug(True) web.run_app(app, port=port, host='0.0.0.0') # app.listen(port=port, address='0.0.0.0') diff --git a/pycrunch/watchdog/connection_watchdog.py b/pycrunch/watchdog/connection_watchdog.py new file mode 100644 index 0000000..dcf5ca3 --- /dev/null +++ b/pycrunch/watchdog/connection_watchdog.py @@ -0,0 +1,57 @@ +import asyncio +import sys + +import logging +from datetime import datetime, timedelta + + +logger = logging.getLogger(__name__) + +# This is to prevent running engine process that Pycharm is no longer using +# In seconds +disconnection_deadline = 100 +significant_sleep_time = timedelta(seconds=disconnection_deadline*2) + + +class ConnectionWatchdog: + def __init__(self): + self.last_wakeup = datetime.now() + self.connected_clients = 0 + self.initial_connection_established = False + + def connection_established(self): + logger.info(f'ConnectionWatchdog->connection_established') + self.connected_clients += 1 + self.initial_connection_established = True + + def connection_lost(self): + logger.info(f'ConnectionWatchdog->connection_lost') + self.connected_clients -= 1 + + async def watch_client_connection_loop(self): + logger.info(f'started ConnectionWatchdog->watch_client_connection_loop') + + # if client was not reconnected in 100 seconds deadline - kill engine + self.last_wakeup = datetime.now() + while True: + logger.debug(f'watch_client_connection_loop-> deciding if we need to close engine') + await asyncio.sleep(disconnection_deadline) + now = datetime.now() + time_interval = now - self.last_wakeup + logger.debug(f' last wakeup {str(time_interval)} ago') + logger.debug(f' at {datetime.now().time()}') + self.last_wakeup = now + if time_interval > significant_sleep_time: + logger.info('watch_client_connection_loop > significant_sleep_time elapsed, will skip next loop') + # Probably PC wake up from sleep mode. + # Sleep one more loop before making decision to terminate + # Allow client plugins more time to reconnect + continue + + if self.initial_connection_established and self.connected_clients <= 0: + logger.warning('!! No connection established within reconnection deadline') + logger.warning('Engine will exit. Please restart it again if you need to.') + sys.exit(1) + + +connection_watchdog = ConnectionWatchdog() From 4f32242f51bab919dbc0145c7ead8dbcfc758cd1 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Thu, 13 Aug 2020 13:48:22 +0300 Subject: [PATCH 5/7] Fixes for windows. Reduced import scope in child runner. --- pycrunch/api/shared.py | 3 --- pycrunch/child_runtime/test_runner.py | 10 ++++------ .../crossprocess/multiprocess_test_runner.py | 19 ++++++++++++++++++- pycrunch/introspection/clock.py | 1 + pycrunch/pipeline/run_test_task.py | 7 ++++--- .../pytest_support/pytest_runner_engine.py | 11 ++++++++--- pycrunch/scheduling/scheduler.py | 3 ++- pycrunch/shared/models.py | 8 -------- pycrunch/shared/primitives.py | 7 +++++++ pycrunch/tests/test_coverage_run.py | 2 +- pycrunch/tests/test_scheduling.py | 3 +-- pycrunch/tests/tests_pinned_tests.py | 3 ++- 12 files changed, 48 insertions(+), 29 deletions(-) create mode 100644 pycrunch/shared/primitives.py diff --git a/pycrunch/api/shared.py b/pycrunch/api/shared.py index 69f3403..c0f682b 100644 --- a/pycrunch/api/shared.py +++ b/pycrunch/api/shared.py @@ -1,6 +1,4 @@ from asyncio import shield -from pprint import pprint -from time import perf_counter import socketio from pycrunch.watcher.fs_watcher import FSWatcher @@ -14,7 +12,6 @@ # log_ws_internals = True log_ws_internals = False sio = socketio.AsyncServer(async_mode=async_mode, cors_allowed_origins='*', logger=log_ws_internals, engineio_logger=log_ws_internals) -timestamp = perf_counter class ExternalPipe: diff --git a/pycrunch/child_runtime/test_runner.py b/pycrunch/child_runtime/test_runner.py index f17495f..7f77426 100644 --- a/pycrunch/child_runtime/test_runner.py +++ b/pycrunch/child_runtime/test_runner.py @@ -1,8 +1,6 @@ from pycrunch.api.serializers import CoverageRun -from pycrunch.child_runtime.child_config import ChildRuntimeConfig from pycrunch.child_runtime.coverage_hal import CoverageAbstraction from pycrunch.insights.variables_inspection import InsightTimeline, inject_timeline -from pycrunch.introspection.clock import clock from pycrunch.runner.execution_result import ExecutionResult DISABLE_COVERAGE = False @@ -15,9 +13,9 @@ def __init__(self, runner_engine, timeline, child_config): def run(self, tests): self.timeline.mark_event('Run: inside run method') - from pycrunch.api.shared import timestamp + from pycrunch.introspection.clock import clock from pycrunch.runner.interception import capture_stdout - from pycrunch.shared.models import TestMetadata + from pycrunch.shared.primitives import TestMetadata self.timeline.mark_event('Run: inside run method - imports complete') results = dict() @@ -43,11 +41,11 @@ def run(self, tests): cov.start() with capture_stdout() as get_value: - time_start = timestamp() + time_start = clock.now() self.timeline.mark_event('About to start test execution') execution_result = self.runner_engine.run_test(metadata) self.timeline.mark_event('Test execution complete, postprocessing results') - time_end = timestamp() + time_end = clock.now() time_elapsed = time_end - time_start cov.stop() diff --git a/pycrunch/crossprocess/multiprocess_test_runner.py b/pycrunch/crossprocess/multiprocess_test_runner.py index e33eced..8baa408 100644 --- a/pycrunch/crossprocess/multiprocess_test_runner.py +++ b/pycrunch/crossprocess/multiprocess_test_runner.py @@ -45,7 +45,9 @@ async def run(self, tests): logger.debug('Initializing subprocess...') child_processes = [] for task in self.tasks: - child_processes.append(asyncio.create_subprocess_shell(self.get_command_line_for_child(port,task.id), cwd=config.working_directory, shell=True)) + shell = True + # child_processes.append(asyncio.create_subprocess_shell(self.get_command_line_for_child(port,task.id), cwd=config.working_directory, shell=shell)) + child_processes.append(asyncio.create_subprocess_exec(sys.executable, *self.get_command_line_for_child_array(port, task.id), cwd=config.working_directory)) logger.debug(f'Waiting for startup of {len(self.tasks)} subprocess task(s)') subprocesses_results = await asyncio.gather(*child_processes) @@ -67,6 +69,8 @@ async def run(self, tests): logger.warning(f'Reached execution timeout of {self.timeout_if_non_debug()} seconds. ') for _ in subprocesses_results: try: + _.terminate() + _.kill() except: logger.warning('Cannot kill child runner process with, ignoring.') @@ -113,6 +117,19 @@ def get_command_line_for_child(self, port, task_id): remote_debug_str = f' --enable-remote-debug --remote-debugger-port={self.remote_debug_params.port}' return sys.executable + hardcoded_path + f'{remote_debug_str}' + def get_command_line_for_child_array(self, port, task_id): + results= [] + results.append(f'{config.engine_directory}{os.sep}pycrunch{os.sep}multiprocess_child_main.py') + results.append(f'--engine={config.runtime_engine}') + results.append(f'--port={port}') + results.append(f'--task-id={task_id}') + results.append(f'--load-pytest-plugins={str(config.load_pytest_plugins).lower()}') + remote_debug_str = '' + if self.remote_debug_params.enabled: + results.append(f'--enable-remote-debug') + results.append(f'--remote-debugger-port={self.remote_debug_params.port}') + return results + def create_server_protocol(self): loop = asyncio.get_event_loop() completion_future = loop.create_future() diff --git a/pycrunch/introspection/clock.py b/pycrunch/introspection/clock.py index 024a063..473b96d 100644 --- a/pycrunch/introspection/clock.py +++ b/pycrunch/introspection/clock.py @@ -3,6 +3,7 @@ class Clock: def now(self): + # floating point value in seconds return time.perf_counter() diff --git a/pycrunch/pipeline/run_test_task.py b/pycrunch/pipeline/run_test_task.py index 2e0d935..fea202f 100644 --- a/pycrunch/pipeline/run_test_task.py +++ b/pycrunch/pipeline/run_test_task.py @@ -5,6 +5,7 @@ from pycrunch.api import shared from pycrunch.api.serializers import CoverageRun from pycrunch.crossprocess.multiprocess_test_runner import MultiprocessTestRunner +from pycrunch.introspection.clock import clock from pycrunch.introspection.history import execution_history from pycrunch.introspection.timings import Timeline from pycrunch.pipeline.abstract_task import AbstractTask @@ -53,7 +54,7 @@ def disabled(cls): class RunTestTask(AbstractTask): def __init__(self, tests, remote_debug_params: RemoteDebugParams): self.remote_debug_params = remote_debug_params - self.timestamp = shared.timestamp() + self.timestamp = clock.now() self.tests = tests self.results = None self.timeline = Timeline('run tests') @@ -100,7 +101,7 @@ async def run(self): async_tasks_post.append(shared.pipe.push( event_type='test_run_completed', coverage=cov_to_send, - timings=dict(start=self.timestamp, end=shared.timestamp()), + timings=dict(start=self.timestamp, end=clock.now()), )) self.timeline.mark_event('Started combined coverage serialization') @@ -115,7 +116,7 @@ async def run(self): # Todo: why do I need dependencies to be exposed? It is internal state. # dependencies=self.build_dependencies(), aggregated_results=engine.all_tests.legacy_aggregated_statuses(), - timings=dict(start=self.timestamp, end=shared.timestamp()), + timings=dict(start=self.timestamp, end=clock.now()), )) self.timeline.mark_event('Waiting until post-processing tasks are completed') diff --git a/pycrunch/plugins/pytest_support/pytest_runner_engine.py b/pycrunch/plugins/pytest_support/pytest_runner_engine.py index 3cde5cc..88ae556 100644 --- a/pycrunch/plugins/pytest_support/pytest_runner_engine.py +++ b/pycrunch/plugins/pytest_support/pytest_runner_engine.py @@ -48,9 +48,14 @@ def run_test(self, test): all_args = additional_pytest_args + plugins_arg # print(all_args, file=sys.__stdout__) if self.child_config.enable_remote_debug: - import pydevd_pycharm - pydevd_pycharm.settrace('127.0.0.1', suspend=False, port=self.child_config.remote_debug_port, stdoutToServer=True, stderrToServer=True) - + try: + import pydevd_pycharm + pydevd_pycharm.settrace('127.0.0.1', suspend=False, port=self.child_config.remote_debug_port, stdoutToServer=True, stderrToServer=True) + except ModuleNotFoundError as e: + print('---\nFailed to import pydevd_pycharm', file=sys.__stdout__) + print(' Make sure you install pudb pycharm bindings by running:', file=sys.__stdout__) + print('pip install pydevd-pycharm\n---', file=sys.__stdout__) + raise pytest.main([fqn_test_to_run] + all_args, plugins=[plugin]) # pytest.main([fqn_test_to_run, '-qs'], plugins=[plugin]) diff --git a/pycrunch/scheduling/scheduler.py b/pycrunch/scheduling/scheduler.py index f7457c8..4d6091d 100644 --- a/pycrunch/scheduling/scheduler.py +++ b/pycrunch/scheduling/scheduler.py @@ -1,4 +1,5 @@ import math +from typing import List from pycrunch.scheduling.sheduled_task import TestRunPlan @@ -15,7 +16,7 @@ def __init__(self, cpu_cores, threshold): self.cpu_cores = cpu_cores self.threshold = threshold - def schedule_into_tasks(self, tests): + def schedule_into_tasks(self, tests) -> List[TestRunPlan]: list_of_tasks = [] total_tests_to_run = len(tests) logger.info(f'total_tests_to_run {total_tests_to_run}') diff --git a/pycrunch/shared/models.py b/pycrunch/shared/models.py index 653b2b1..9112a2d 100644 --- a/pycrunch/shared/models.py +++ b/pycrunch/shared/models.py @@ -6,14 +6,6 @@ logger = logging.getLogger(__name__) -class TestMetadata: - def __init__(self, filename, name, module, fqn, state): - self.state = state - self.fqn = fqn - self.module = module - self.name = name - self.filename = filename - class TestState: def __init__(self, discovered_test, execution_result, pinned): self.discovered_test = discovered_test diff --git a/pycrunch/shared/primitives.py b/pycrunch/shared/primitives.py new file mode 100644 index 0000000..7465800 --- /dev/null +++ b/pycrunch/shared/primitives.py @@ -0,0 +1,7 @@ +class TestMetadata: + def __init__(self, filename, name, module, fqn, state): + self.state = state + self.fqn = fqn + self.module = module + self.name = name + self.filename = filename diff --git a/pycrunch/tests/test_coverage_run.py b/pycrunch/tests/test_coverage_run.py index 2f05584..8fe92c6 100644 --- a/pycrunch/tests/test_coverage_run.py +++ b/pycrunch/tests/test_coverage_run.py @@ -6,7 +6,7 @@ from pycrunch.insights.variables_inspection import InsightTimeline, trace from pycrunch.introspection.clock import Clock from pycrunch.runner.execution_result import ExecutionResult -from pycrunch.shared.models import TestMetadata +from pycrunch.shared.primitives import TestMetadata class TestCoverageRun(unittest.TestCase): diff --git a/pycrunch/tests/test_scheduling.py b/pycrunch/tests/test_scheduling.py index ef8a9e3..db480f7 100644 --- a/pycrunch/tests/test_scheduling.py +++ b/pycrunch/tests/test_scheduling.py @@ -1,8 +1,7 @@ import uuid from pycrunch.scheduling.scheduler import TestRunScheduler -from pycrunch.scheduling.sheduled_task import TestRunPlan -from pycrunch.shared.models import TestMetadata +from pycrunch.shared.primitives import TestMetadata def generate_test_fake(name): filename = str(uuid.uuid4()) diff --git a/pycrunch/tests/tests_pinned_tests.py b/pycrunch/tests/tests_pinned_tests.py index 09535b2..6960bb5 100644 --- a/pycrunch/tests/tests_pinned_tests.py +++ b/pycrunch/tests/tests_pinned_tests.py @@ -4,7 +4,8 @@ from pycrunch.runner.execution_result import ExecutionResult from pycrunch.session.configuration import Configuration from pycrunch.session.state import engine -from pycrunch.shared.models import TestState, TestMetadata, AllTests +from pycrunch.shared.models import TestState, AllTests +from pycrunch.shared.primitives import TestMetadata def test_pinned_test_should_change_status(): From b2833671578e35901b764d2c75838070156acad0 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Thu, 13 Aug 2020 15:42:09 +0300 Subject: [PATCH 6/7] Removed dead code regarding shell executor --- pycrunch/api/socket_handlers.py | 4 ++-- .../crossprocess/multiprocess_test_runner.py | 21 ++++++------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pycrunch/api/socket_handlers.py b/pycrunch/api/socket_handlers.py index f221680..cfa807e 100644 --- a/pycrunch/api/socket_handlers.py +++ b/pycrunch/api/socket_handlers.py @@ -31,12 +31,12 @@ def handle_message(message): @shared.sio.on('json') -def handle_json(json): +async def handle_json(json, smth): logger.debug('handle_json') # logger.debug(session['userid']) # url_for1 = url_for('my event', _external=True) # logger.debug('url + ' + url_for1) - pipe.push(event_type='connected', **{'data': 'Connected'}) + await pipe.push(event_type='connected', **{'data': 'Connected'}) logger.debug('received json 2: ' + str(json)) diff --git a/pycrunch/crossprocess/multiprocess_test_runner.py b/pycrunch/crossprocess/multiprocess_test_runner.py index 8baa408..1da70de 100644 --- a/pycrunch/crossprocess/multiprocess_test_runner.py +++ b/pycrunch/crossprocess/multiprocess_test_runner.py @@ -45,9 +45,7 @@ async def run(self, tests): logger.debug('Initializing subprocess...') child_processes = [] for task in self.tasks: - shell = True - # child_processes.append(asyncio.create_subprocess_shell(self.get_command_line_for_child(port,task.id), cwd=config.working_directory, shell=shell)) - child_processes.append(asyncio.create_subprocess_exec(sys.executable, *self.get_command_line_for_child_array(port, task.id), cwd=config.working_directory)) + child_processes.append(self.create_child_subprocess(port, task)) logger.debug(f'Waiting for startup of {len(self.tasks)} subprocess task(s)') subprocesses_results = await asyncio.gather(*child_processes) @@ -69,8 +67,6 @@ async def run(self, tests): logger.warning(f'Reached execution timeout of {self.timeout_if_non_debug()} seconds. ') for _ in subprocesses_results: try: - _.terminate() - _.kill() except: logger.warning('Cannot kill child runner process with, ignoring.') @@ -104,27 +100,22 @@ async def run(self, tests): raise asyncio.TimeoutError('Test execution timeout.') return _results + def create_child_subprocess(self, port, task): + # sys.executable is a full path to python.exe (or ./python) in current virtual environment + return asyncio.create_subprocess_exec(sys.executable, *self.get_command_line_for_child(port, task.id), cwd=config.working_directory) + def timeout_if_non_debug(self) -> Optional[float]: if self.remote_debug_params.enabled: return None return self.timeout def get_command_line_for_child(self, port, task_id): - engine_root = f' {config.engine_directory}{os.sep}pycrunch{os.sep}multiprocess_child_main.py ' - hardcoded_path = engine_root + f'--engine={config.runtime_engine} --port={port} --task-id={task_id} --load-pytest-plugins={str(config.load_pytest_plugins).lower()}' - remote_debug_str = '' - if self.remote_debug_params.enabled: - remote_debug_str = f' --enable-remote-debug --remote-debugger-port={self.remote_debug_params.port}' - return sys.executable + hardcoded_path + f'{remote_debug_str}' - - def get_command_line_for_child_array(self, port, task_id): - results= [] + results = [] results.append(f'{config.engine_directory}{os.sep}pycrunch{os.sep}multiprocess_child_main.py') results.append(f'--engine={config.runtime_engine}') results.append(f'--port={port}') results.append(f'--task-id={task_id}') results.append(f'--load-pytest-plugins={str(config.load_pytest_plugins).lower()}') - remote_debug_str = '' if self.remote_debug_params.enabled: results.append(f'--enable-remote-debug') results.append(f'--remote-debugger-port={self.remote_debug_params.port}') From 8c1e00a8fb634a80449e59e1013168bf4de0e9e7 Mon Sep 17 00:00:00 2001 From: Gleb Sevruk Date: Tue, 18 Aug 2020 17:53:16 +0300 Subject: [PATCH 7/7] Release url and version changed --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 311cd46..6854025 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup setup(name='pycrunch-engine', - version='1.1', + version='1.2', description='Automatic Test Runner Engine', url='http://github.com/gleb-sevruk/pycrunch-engine', author='Gleb Sevruk', @@ -10,7 +10,7 @@ license='libpng', keywords="tdd unit-testing test runner", packages=setuptools.find_packages(), - download_url='https://github.com/gleb-sevruk/pycrunch-engine/archive/v1.1.tar.gz', + download_url='https://github.com/gleb-sevruk/pycrunch-engine/archive/v1.2.tar.gz', setup_requires=['wheel'], entry_points={ 'console_scripts': ['pycrunch-engine=pycrunch.main:run'],