Skip to content

Commit

Permalink
Finished PM logs
Browse files Browse the repository at this point in the history
  • Loading branch information
PawelPlesniak committed Nov 19, 2024
1 parent 06f1629 commit 963cc0a
Show file tree
Hide file tree
Showing 6 changed files with 31 additions and 25 deletions.
2 changes: 1 addition & 1 deletion src/drunc/process_manager/interface/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
@click.command('boot')
@click.option('-u','--user', type=str, default=getpass.getuser(), help='Select the process of a particular user (default $USER)')
@click.option('-l', '--log-level', type=click.Choice(log_levels.keys(), case_sensitive=False), default='INFO', help='Set the log level')
@click.option('--override-logs/--no-override-logs', default=True)
@click.option('-o/-no', '--override-logs/--no-override-logs', type=bool, default=True, help="Override logs, if --no-override-logs filenames have the timestamp of the run.")
@click.argument('boot-configuration', type=str, callback=validate_conf_string)
@click.argument('session-name', type=str)
@click.pass_obj
Expand Down
26 changes: 15 additions & 11 deletions src/drunc/process_manager/interface/process_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@
import os
import logging
import getpass

from drunc.utils.utils import log_levels
from rich.console import Console
from rich.logging import RichHandler
from drunc.utils.utils import log_levels, update_log_level

_cleanup_coroutines = []

def run_pm(pm_conf:str, pm_address:str, override_logs:bool, log_level:str, ready_event:bool=None, signal_handler:bool=None, generated_port:bool=None):
log = logging.getLogger("run_pm")
log.debug("Running run_pm")
log.addHandler(RichHandler())
update_log_level(log_level)
log.info("Running run_pm")
if signal_handler is not None:
signal_handler()

Expand Down Expand Up @@ -46,7 +49,6 @@ async def serve(address:str) -> None:
from drunc.exceptions import DruncSetupException
raise DruncSetupException('The address on which to expect commands/send status wasn\'t specified')
from druncschema.process_manager_pb2_grpc import add_ProcessManagerServicer_to_server

server = grpc.aio.server()
add_ProcessManagerServicer_to_server(pm, server)
port = server.add_insecure_port(address)
Expand All @@ -56,7 +58,7 @@ async def serve(address:str) -> None:
await server.start()
import socket
hostname = socket.gethostname()
log.debug(f'ProcessManager was started on {hostname}:{port}')
log.info(f'ProcessManager was started on {hostname}:{port}')


async def server_shutdown():
Expand All @@ -65,12 +67,11 @@ async def server_shutdown():
# grace period, the server won't accept new connections and allow
# existing RPCs to continue within the grace period.
await server.stop(5)
pm._terminate_impl(None)
pm._terminate_impl()

_cleanup_coroutines.append(server_shutdown())
if ready_event is not None:
ready_event.set()

await server.wait_for_termination()


Expand All @@ -82,27 +83,30 @@ async def server_shutdown():
except Exception as e:
log.error("Serving the ProcessManager received an Exception")
import os
from rich.console import Console
console = Console()
console.print_exception(width=os.get_terminal_size()[0])
finally:
if _cleanup_coroutines:
log.debug("Clearing coroutines")
log.info("Clearing coroutines")
loop.run_until_complete(*_cleanup_coroutines)
loop.close()

@click.command()
@click.argument('pm-conf', type=str)
@click.argument('pm-port', type=int)
@click.option('-o/-no', '--override-logs/--no-override-logs', type=bool, default=True, help="Override logs, if --no-override-logs filenames have the timestamp of the run.")
@click.option(
'-l',
'--log-level',
type=click.Choice(
log_levels.keys(),
case_sensitive=False
),
default='INFO',
default='DEBUG',
help='Set the log level'
)
def process_manager_cli(pm_conf:str, pm_port:int, log_level):
def process_manager_cli(pm_conf:str, pm_port:int, override_logs:bool, log_level:str):
from drunc.process_manager.configuration import get_process_manager_configuration
pm_conf = get_process_manager_configuration(pm_conf)
run_pm(pm_conf, f'0.0.0.0:{pm_port}', log_level)
run_pm(pm_conf, f'0.0.0.0:{pm_port}', override_logs, log_level)
12 changes: 8 additions & 4 deletions src/drunc/process_manager/process_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,6 @@ class ProcessManager(abc.ABC, ProcessManagerServicer):

def __init__(self, configuration:ProcessManagerConfHandler, name:str, override_logs:bool, log_level:str, session=None, **kwargs):
super().__init__()

self.log = logging.getLogger("process_manager")
self.log.debug(pid_info_str())
log_path = get_log_path(
user = kwargs.get("user", getpass.getuser()),
session_name = type(self).__name__,
Expand All @@ -38,11 +35,18 @@ def __init__(self, configuration:ProcessManagerConfHandler, name:str, override_l
if override_logs and os.path.isfile(log_path):
os.remove(log_path)

self.log = logging.getLogger("process_manager")
handler = logging.FileHandler(log_path)
handler.setLevel(log_level)
formatter = logging.Formatter("%(asctime)s[%(levelname)s] %(funcName)s: %(message)s", "[%H:%M:%S]")
formatter = logging.Formatter(
fmt = "%(asctime)s\t%(levelname)s\t%(filename)s:%(lineno)i\t%(name)s:\t%(message)s",
datefmt = "[%X]",
style="%"
)
handler.setFormatter(formatter)
self.log.addHandler(handler)

self.log.debug(pid_info_str())
self.log.debug("Initialized ProcessManager")

self.configuration = configuration
Expand Down
13 changes: 6 additions & 7 deletions src/drunc/process_manager/process_manager_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,13 +158,13 @@ async def boot(
from daqconf.consolidate import consolidate_db
consolidate_db(oks_conf, f"{fname}")
except Exception as e:
log.critical(f'''\nInvalid configuration passed (cannot consolidate your configuration)
{e}
To debug it, close drunc and run the following command:
self._log.critical(f'''\nInvalid configuration passed (cannot consolidate your configuration)
{e}
To debug it, close drunc and run the following command:
[yellow]oks_dump --files-only {oks_conf}[/]
[yellow]oks_dump --files-only {oks_conf}[/]
''', extra={'markup': True})
''', extra={'markup': True})
return

db = conffwk.Configuration(f"oksconflibs:{oks_conf}")
Expand Down Expand Up @@ -252,8 +252,7 @@ def get_controller_address(session_dal, session_name):

import signal
def keyboard_interrupt_on_sigint(signal, frame):
from logging import getLogger
log.warning("Interrupted")
self._log.warning("Interrupted")
raise KeyboardInterrupt

original_sigint_handler = signal.getsignal(signal.SIGINT)
Expand Down
2 changes: 1 addition & 1 deletion src/drunc/process_manager/ssh_process_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ def kill_processes(self, uuids:list) -> ProcessInstanceList:


def _terminate_impl(self) -> ProcessInstanceList:
self.log.debug(f'{type(self).__name__} terminating')
self.log.warning(f'{type(self).__name__} terminating')
if self.process_store:
self.log.warning('Killing all the known processes before exiting')
uuids = [uuid for uuid, process in self.process_store.items()]
Expand Down
1 change: 0 additions & 1 deletion src/drunc/utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ def print_traceback():

def update_log_level(loglevel):
log_level = log_levels[loglevel]
#logging.basicConfig(level=log_level)
# Update log level for root logger
logger = logging.getLogger('drunc')
logger.setLevel(log_level)
Expand Down

0 comments on commit 963cc0a

Please sign in to comment.