From 963cc0ab4aac8dab0cf5717f37281e84769dc4b0 Mon Sep 17 00:00:00 2001 From: PawelPlesniak Date: Tue, 19 Nov 2024 20:55:09 +0100 Subject: [PATCH] Finished PM logs --- .../process_manager/interface/commands.py | 2 +- .../interface/process_manager.py | 26 +++++++++++-------- src/drunc/process_manager/process_manager.py | 12 ++++++--- .../process_manager/process_manager_driver.py | 13 +++++----- .../process_manager/ssh_process_manager.py | 2 +- src/drunc/utils/utils.py | 1 - 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/drunc/process_manager/interface/commands.py b/src/drunc/process_manager/interface/commands.py index 8a7b46f0..721aa878 100644 --- a/src/drunc/process_manager/interface/commands.py +++ b/src/drunc/process_manager/interface/commands.py @@ -12,7 +12,7 @@ @click.command('boot') @click.option('-u','--user', type=str, default=getpass.getuser(), help='Select the process of a particular user (default $USER)') @click.option('-l', '--log-level', type=click.Choice(log_levels.keys(), case_sensitive=False), default='INFO', help='Set the log level') -@click.option('--override-logs/--no-override-logs', default=True) +@click.option('-o/-no', '--override-logs/--no-override-logs', type=bool, default=True, help="Override logs, if --no-override-logs filenames have the timestamp of the run.") @click.argument('boot-configuration', type=str, callback=validate_conf_string) @click.argument('session-name', type=str) @click.pass_obj diff --git a/src/drunc/process_manager/interface/process_manager.py b/src/drunc/process_manager/interface/process_manager.py index adc17937..2af3cc05 100644 --- a/src/drunc/process_manager/interface/process_manager.py +++ b/src/drunc/process_manager/interface/process_manager.py @@ -4,14 +4,17 @@ import os import logging import getpass - -from drunc.utils.utils import log_levels +from rich.console import Console +from rich.logging import RichHandler +from drunc.utils.utils import log_levels, update_log_level _cleanup_coroutines = [] def run_pm(pm_conf:str, pm_address:str, override_logs:bool, log_level:str, ready_event:bool=None, signal_handler:bool=None, generated_port:bool=None): log = logging.getLogger("run_pm") - log.debug("Running run_pm") + log.addHandler(RichHandler()) + update_log_level(log_level) + log.info("Running run_pm") if signal_handler is not None: signal_handler() @@ -46,7 +49,6 @@ async def serve(address:str) -> None: from drunc.exceptions import DruncSetupException raise DruncSetupException('The address on which to expect commands/send status wasn\'t specified') from druncschema.process_manager_pb2_grpc import add_ProcessManagerServicer_to_server - server = grpc.aio.server() add_ProcessManagerServicer_to_server(pm, server) port = server.add_insecure_port(address) @@ -56,7 +58,7 @@ async def serve(address:str) -> None: await server.start() import socket hostname = socket.gethostname() - log.debug(f'ProcessManager was started on {hostname}:{port}') + log.info(f'ProcessManager was started on {hostname}:{port}') async def server_shutdown(): @@ -65,12 +67,11 @@ async def server_shutdown(): # grace period, the server won't accept new connections and allow # existing RPCs to continue within the grace period. await server.stop(5) - pm._terminate_impl(None) + pm._terminate_impl() _cleanup_coroutines.append(server_shutdown()) if ready_event is not None: ready_event.set() - await server.wait_for_termination() @@ -82,16 +83,19 @@ async def server_shutdown(): except Exception as e: log.error("Serving the ProcessManager received an Exception") import os + from rich.console import Console + console = Console() console.print_exception(width=os.get_terminal_size()[0]) finally: if _cleanup_coroutines: - log.debug("Clearing coroutines") + log.info("Clearing coroutines") loop.run_until_complete(*_cleanup_coroutines) loop.close() @click.command() @click.argument('pm-conf', type=str) @click.argument('pm-port', type=int) +@click.option('-o/-no', '--override-logs/--no-override-logs', type=bool, default=True, help="Override logs, if --no-override-logs filenames have the timestamp of the run.") @click.option( '-l', '--log-level', @@ -99,10 +103,10 @@ async def server_shutdown(): log_levels.keys(), case_sensitive=False ), - default='INFO', + default='DEBUG', help='Set the log level' ) -def process_manager_cli(pm_conf:str, pm_port:int, log_level): +def process_manager_cli(pm_conf:str, pm_port:int, override_logs:bool, log_level:str): from drunc.process_manager.configuration import get_process_manager_configuration pm_conf = get_process_manager_configuration(pm_conf) - run_pm(pm_conf, f'0.0.0.0:{pm_port}', log_level) + run_pm(pm_conf, f'0.0.0.0:{pm_port}', override_logs, log_level) \ No newline at end of file diff --git a/src/drunc/process_manager/process_manager.py b/src/drunc/process_manager/process_manager.py index 52d79845..89eb9da2 100644 --- a/src/drunc/process_manager/process_manager.py +++ b/src/drunc/process_manager/process_manager.py @@ -26,9 +26,6 @@ class ProcessManager(abc.ABC, ProcessManagerServicer): def __init__(self, configuration:ProcessManagerConfHandler, name:str, override_logs:bool, log_level:str, session=None, **kwargs): super().__init__() - - self.log = logging.getLogger("process_manager") - self.log.debug(pid_info_str()) log_path = get_log_path( user = kwargs.get("user", getpass.getuser()), session_name = type(self).__name__, @@ -38,11 +35,18 @@ def __init__(self, configuration:ProcessManagerConfHandler, name:str, override_l if override_logs and os.path.isfile(log_path): os.remove(log_path) + self.log = logging.getLogger("process_manager") handler = logging.FileHandler(log_path) handler.setLevel(log_level) - formatter = logging.Formatter("%(asctime)s[%(levelname)s] %(funcName)s: %(message)s", "[%H:%M:%S]") + formatter = logging.Formatter( + fmt = "%(asctime)s\t%(levelname)s\t%(filename)s:%(lineno)i\t%(name)s:\t%(message)s", + datefmt = "[%X]", + style="%" + ) handler.setFormatter(formatter) self.log.addHandler(handler) + + self.log.debug(pid_info_str()) self.log.debug("Initialized ProcessManager") self.configuration = configuration diff --git a/src/drunc/process_manager/process_manager_driver.py b/src/drunc/process_manager/process_manager_driver.py index 669674b4..4c83f33e 100644 --- a/src/drunc/process_manager/process_manager_driver.py +++ b/src/drunc/process_manager/process_manager_driver.py @@ -158,13 +158,13 @@ async def boot( from daqconf.consolidate import consolidate_db consolidate_db(oks_conf, f"{fname}") except Exception as e: - log.critical(f'''\nInvalid configuration passed (cannot consolidate your configuration) - {e} - To debug it, close drunc and run the following command: + self._log.critical(f'''\nInvalid configuration passed (cannot consolidate your configuration) + {e} + To debug it, close drunc and run the following command: - [yellow]oks_dump --files-only {oks_conf}[/] + [yellow]oks_dump --files-only {oks_conf}[/] - ''', extra={'markup': True}) + ''', extra={'markup': True}) return db = conffwk.Configuration(f"oksconflibs:{oks_conf}") @@ -252,8 +252,7 @@ def get_controller_address(session_dal, session_name): import signal def keyboard_interrupt_on_sigint(signal, frame): - from logging import getLogger - log.warning("Interrupted") + self._log.warning("Interrupted") raise KeyboardInterrupt original_sigint_handler = signal.getsignal(signal.SIGINT) diff --git a/src/drunc/process_manager/ssh_process_manager.py b/src/drunc/process_manager/ssh_process_manager.py index 26ecbe2d..9fc9705e 100644 --- a/src/drunc/process_manager/ssh_process_manager.py +++ b/src/drunc/process_manager/ssh_process_manager.py @@ -130,7 +130,7 @@ def kill_processes(self, uuids:list) -> ProcessInstanceList: def _terminate_impl(self) -> ProcessInstanceList: - self.log.debug(f'{type(self).__name__} terminating') + self.log.warning(f'{type(self).__name__} terminating') if self.process_store: self.log.warning('Killing all the known processes before exiting') uuids = [uuid for uuid, process in self.process_store.items()] diff --git a/src/drunc/utils/utils.py b/src/drunc/utils/utils.py index eb7c3f99..7f3ea6b3 100644 --- a/src/drunc/utils/utils.py +++ b/src/drunc/utils/utils.py @@ -44,7 +44,6 @@ def print_traceback(): def update_log_level(loglevel): log_level = log_levels[loglevel] - #logging.basicConfig(level=log_level) # Update log level for root logger logger = logging.getLogger('drunc') logger.setLevel(log_level)