From 552b053b3b66f74fdb383749d30db5d284c87760 Mon Sep 17 00:00:00 2001 From: Dhanshree Arora Date: Fri, 26 Jul 2024 17:30:57 +0300 Subject: [PATCH] Enable Ersilia to serve multiple models simultaneously (#1201) * WIP Add session management utils * Create a session at ersilia cli initialization * Remove unused code * start an ersilia session in a dedicated sessions dir that is mapped to the parent process' id which ran the given ersilia command, this will generally be a shell process inside a terminal, but it can also be a process from a bash script * declare session specific defaults * Run all ersilia commands within a single process during standard example run, otherwise ersilia run command does not find a served model bec of running in a different process and therefore in a different session * Move the currently served model's pid to its dedicated session directory * WIP Logging * Redirect tmp logs to model's session logs * catch permission error * Redirect tmp logs to model's session logs * don't use ersilia exception management because we don't exactly want the ersilia process to exit --- ersilia/cli/__init__.py | 3 +- ersilia/cli/commands/utils/utils.py | 7 --- ersilia/cli/echo.py | 5 +- ersilia/core/model.py | 3 +- ersilia/core/session.py | 6 ++- ersilia/db/environments/managers.py | 19 +++---- ersilia/default.py | 18 +++++-- ersilia/hub/content/card.py | 3 +- ersilia/hub/delete/delete.py | 5 +- ersilia/hub/fetch/actions/get.py | 3 +- ersilia/hub/fetch/actions/modify.py | 5 +- .../hub/fetch/inner_template/src/service.py | 3 +- ersilia/hub/fetch/lazy_fetchers/dockerhub.py | 10 ++-- .../hub/fetch/pack/bentoml_pack/runners.py | 3 +- .../hub/fetch/register/standard_example.py | 5 +- ersilia/hub/pull/pull.py | 3 +- ersilia/io/output.py | 3 +- ersilia/io/readers/file.py | 8 +-- ersilia/publish/metadata.py | 5 +- ersilia/publish/s3.py | 5 +- ersilia/publish/test.py | 3 +- ersilia/serve/api.py | 7 +-- ersilia/serve/services.py | 20 ++++--- ersilia/setup/baseconda.py | 5 +- ersilia/setup/basedocker.py | 3 +- ersilia/setup/requirements/bentoml.py | 2 +- ersilia/utils/__init__.py | 6 ++- ersilia/utils/conda.py | 19 +++---- ersilia/utils/docker.py | 9 ++-- ersilia/utils/download.py | 8 +-- ersilia/utils/exceptions_utils/exceptions.py | 3 +- ersilia/utils/installers.py | 5 +- ersilia/utils/logging.py | 32 +++++++---- ersilia/utils/session.py | 54 +++++++++++++++++++ ersilia/utils/terminal.py | 9 ++-- ersilia/utils/venv.py | 3 +- 36 files changed, 208 insertions(+), 102 deletions(-) delete mode 100644 ersilia/cli/commands/utils/utils.py create mode 100644 ersilia/utils/session.py diff --git a/ersilia/cli/__init__.py b/ersilia/cli/__init__.py index dde6a9ad2..711999b04 100644 --- a/ersilia/cli/__init__.py +++ b/ersilia/cli/__init__.py @@ -1,7 +1,8 @@ from .create_cli import create_ersilia_cli from .echo import echo +from ..utils.session import create_session_dir cli = create_ersilia_cli() - +create_session_dir() if __name__ == "__main__": cli() diff --git a/ersilia/cli/commands/utils/utils.py b/ersilia/cli/commands/utils/utils.py deleted file mode 100644 index 0c4fa858c..000000000 --- a/ersilia/cli/commands/utils/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -import os -from ....default import EOS - - -def tmp_pid_file(model_id): - """Gets filename to store process ID""" - return os.path.join(EOS, "tmp", "{0}.pid".format(model_id)) diff --git a/ersilia/cli/echo.py b/ersilia/cli/echo.py index 58fb42c6b..270f3497c 100644 --- a/ersilia/cli/echo.py +++ b/ersilia/cli/echo.py @@ -5,12 +5,13 @@ import click import os import json -from ..default import EOS, SILENCE_FILE +from ..default import SILENCE_FILE +from ..utils.session import get_session_dir class Silencer(object): def __init__(self): - self.silence_file = os.path.join(EOS, SILENCE_FILE) + self.silence_file = os.path.join(get_session_dir(), SILENCE_FILE) if not os.path.exists(self.silence_file): self.speak() diff --git a/ersilia/core/model.py b/ersilia/core/model.py index b640d38a3..615a5b557 100644 --- a/ersilia/core/model.py +++ b/ersilia/core/model.py @@ -29,6 +29,7 @@ from ..utils.exceptions_utils.api_exceptions import ApiSpecifiedOutputError from ..default import FETCHED_MODELS_FILENAME, MODEL_SIZE_FILE, CARD_FILE, EOS from ..default import DEFAULT_BATCH_SIZE, APIS_LIST_FILE, INFORMATION_FILE +from ..utils.logging import make_temp_dir try: import pandas as pd @@ -222,7 +223,7 @@ def _api_runner_return(self, api, input, output, batch_size): R += [r] return json.dumps(R, indent=4) else: - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") is_h5_serializable = self.api_schema.is_h5_serializable( api_name=api.api_name ) diff --git a/ersilia/core/session.py b/ersilia/core/session.py index 42d0a7255..f7aa9350b 100644 --- a/ersilia/core/session.py +++ b/ersilia/core/session.py @@ -4,15 +4,17 @@ import time import uuid import shutil +from ..utils.session import get_session_dir -from ..default import EOS +from ..default import SESSIONS_DIR from .base import ErsiliaBase class Session(ErsiliaBase): def __init__(self, config_json): ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) - self.session_file = os.path.join(EOS, "session.json") + session_dir = get_session_dir() + self.session_file = os.path.join(session_dir, "session.json") def current_model_id(self): data = self.get() diff --git a/ersilia/db/environments/managers.py b/ersilia/db/environments/managers.py index 53dfab29e..bf9fe3def 100644 --- a/ersilia/db/environments/managers.py +++ b/ersilia/db/environments/managers.py @@ -12,8 +12,9 @@ from ...utils.identifiers.short import ShortIdentifier from ...utils.ports import find_free_port from .localdb import EnvironmentDb - from ...default import DOCKERHUB_ORG, DOCKERHUB_LATEST_TAG +from ...utils.session import get_session_dir +from ...utils.logging import make_temp_dir import sys @@ -99,7 +100,7 @@ def containers_of_model(self, model_id, only_run, only_latest=True): def build_with_bentoml(self, model_id, use_cache=True): bundle_path = self._get_bundle_location(model_id) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "build.sh") cmdlines = ["cd {0}".format(bundle_path)] if use_cache: @@ -128,7 +129,7 @@ def _model_deploy_dockerfiles_url(self): def _build_ersilia_base(self): self.logger.debug("Creating docker image of ersilia base") - path = tempfile.mkdtemp(prefix="ersilia-") + path = make_temp_dir(prefix="ersilia-") base_folder = os.path.join(path, "base") os.mkdir(base_folder) base_files = [ @@ -153,7 +154,7 @@ def build_with_ersilia(self, model_id, docker_user, docker_pwd): pass else: self._build_ersilia_base() - path = tempfile.mkdtemp(prefix="ersilia-model") + path = make_temp_dir(prefix="ersilia-model") model_folder = os.path.join(path, model_id) os.mkdir(model_folder) cmd = "cd {0}; wget {1}/model/Dockerfile".format( @@ -281,7 +282,7 @@ def _stop_containers_with_model_id(self, model_id): return if not self.is_installed(): return - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "docker-ps.txt") cmd = "docker ps > {0}".format(tmp_file) self.logger.debug("Running {0}".format(cmd)) @@ -305,7 +306,7 @@ def _stop_containers_with_entrypoint_sh(self): return if not self.is_installed(): return - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "docker-ps.txt") cmd = "docker ps > {0}".format(tmp_file) self.logger.debug("Running {0}".format(cmd)) @@ -332,7 +333,7 @@ def stop_containers(self, model_id): self._stop_containers_with_model_id(model_id) self._stop_containers_with_entrypoint_sh() self.remove_stopped_containers() - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "docker-ps.txt") cmd = "docker ps > {0}".format(tmp_file) self.logger.debug("Running {0}".format(cmd)) @@ -356,7 +357,7 @@ def prune(self): run_command(cmd) def delete_image(self, img): - fn = os.path.join(self._tmp_dir, "rm_image_output.txt") + fn = os.path.join(get_session_dir(), "rm_image_output.txt") cmd = "docker image rm {0} --force 2> {1}".format(img, fn) run_command(cmd) with open(fn, "r") as f: @@ -382,7 +383,7 @@ def delete_images(self, model_id, purge_unnamed=True): return self.stop_containers(model_id) self.prune() - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "docker-images.txt") cmd = "docker images > {0}".format(tmp_file) self.logger.debug("Running {0}".format(cmd)) diff --git a/ersilia/default.py b/ersilia/default.py index 0b21f0afd..8ba1198bb 100644 --- a/ersilia/default.py +++ b/ersilia/default.py @@ -28,11 +28,7 @@ DEFAULT_VENV = "env" DEFAULT_API_NAME = "run" PACKMODE_FILE = "pack_mode.txt" -LOGGING_FILE = "console.log" -CURRENT_LOGGING_FILE = "current.log" CARD_FILE = "card.json" -SILENCE_FILE = ".silence.json" -VERBOSE_FILE = ".verbose.json" API_SCHEMA_FILE = "api_schema.json" MODEL_SIZE_FILE = "size.json" DEFAULT_BATCH_SIZE = 100 @@ -61,6 +57,20 @@ PACK_METHOD_FASTAPI = "fastapi" PACK_METHOD_BENTOML = "bentoml" +# Session and logging +SESSIONS_DIR = os.path.join(EOS, "sessions") +if not os.path.exists(SESSIONS_DIR): + os.makedirs(SESSIONS_DIR, exist_ok=True) +SESSION_HISTORY_FILE = "history.txt" +SESSION_JSON = "session.json" +LOGS_DIR = "logs" +CONTAINER_LOGS_TMP_DIR = "_logs/tmp" +CONTAINER_LOGS_EOS_DIR = "_logs/eos" # This is not used +LOGGING_FILE = "console.log" +CURRENT_LOGGING_FILE = "current.log" +SILENCE_FILE = ".silence.json" +VERBOSE_FILE = ".verbose.json" + # Isaura data lake H5_EXTENSION = ".h5" H5_DATA_FILE = "data.h5" diff --git a/ersilia/hub/content/card.py b/ersilia/hub/content/card.py index e9e848fed..0f56bd282 100644 --- a/ersilia/hub/content/card.py +++ b/ersilia/hub/content/card.py @@ -41,6 +41,7 @@ MemoryGbBaseInformationError, ) from ...utils.identifiers.model import ModelIdentifier +from ...utils.logging import make_temp_dir try: from isaura.core.hdf5 import Hdf5Explorer @@ -622,7 +623,7 @@ def _raw_readme_url(self, model_id): return url def _gh_view(self, model_id): - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "view.md") cmd = "gh repo view {0}/{1} > {2}".format("ersilia-os", model_id, tmp_file) run_command(cmd) diff --git a/ersilia/hub/delete/delete.py b/ersilia/hub/delete/delete.py index 0f2c505d5..bb8c7f7c8 100644 --- a/ersilia/hub/delete/delete.py +++ b/ersilia/hub/delete/delete.py @@ -232,8 +232,9 @@ def __init__(self, config_json=None): ErsiliaBase.__init__(self, config_json=config_json) def delete(self): - os.rmdir(self._tmp_dir) - os.makedirs(self._tmp_dir) + if os.path.exists(self._tmp_dir): + os.rmdir(self._tmp_dir) + os.makedirs(self._tmp_dir) class ModelFullDeleter(ErsiliaBase): diff --git a/ersilia/hub/fetch/actions/get.py b/ersilia/hub/fetch/actions/get.py index 6a6e8ac28..e47661025 100644 --- a/ersilia/hub/fetch/actions/get.py +++ b/ersilia/hub/fetch/actions/get.py @@ -18,6 +18,7 @@ from .template_resolver import TemplateResolver from ....default import S3_BUCKET_URL_ZIP, PREDEFINED_EXAMPLE_FILES +from ....utils.logging import make_temp_dir MODEL_DIR = "model" ROOT = os.path.basename(os.path.abspath(__file__)) @@ -207,7 +208,7 @@ def _copy_from_github(self, dst): def _copy_zip_from_s3(self, dst): self.logger.debug("Downloading model from S3 in zipped format") - tmp_file = os.path.join(tempfile.mkdtemp("ersilia-"), "model.zip") + tmp_file = os.path.join(make_temp_dir("ersilia-"), "model.zip") self.s3_down.download_from_s3( bucket_url=S3_BUCKET_URL_ZIP, file_name=self.model_id + ".zip", diff --git a/ersilia/hub/fetch/actions/modify.py b/ersilia/hub/fetch/actions/modify.py index 92d7a57ce..0522189c8 100644 --- a/ersilia/hub/fetch/actions/modify.py +++ b/ersilia/hub/fetch/actions/modify.py @@ -13,6 +13,7 @@ BundleRequirementsFile, ) from ...bundle.repo import DockerfileFile +from ....utils.logging import make_temp_dir class ModelModifier(BaseAction): @@ -24,7 +25,7 @@ def __init__(self, model_id, config_json): def _bundle_uses_ersilia(self, model_id): """Check if the bundle imports ersilia""" src = os.path.join(self._get_bundle_location(model_id), model_id, "src") - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "grep.txt") cmd = "grep -R 'ersilia' {0}/* > {1}".format(src, tmp_file) run_command(cmd) @@ -86,7 +87,7 @@ def _bundle_dockerfile_has_ersilia(self, model_id): dockerfile = os.path.join(dir, DOCKERFILE) if not os.path.exists(dockerfile): return None - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "grep.txt") cmd = "grep -R 'ersilia' {0} > {1}".format(dockerfile, tmp_file) run_command(cmd) diff --git a/ersilia/hub/fetch/inner_template/src/service.py b/ersilia/hub/fetch/inner_template/src/service.py index 4ef78dc71..9be35f81b 100644 --- a/ersilia/hub/fetch/inner_template/src/service.py +++ b/ersilia/hub/fetch/inner_template/src/service.py @@ -12,6 +12,7 @@ import subprocess import csv +from .....utils.logging import make_temp_dir CHECKPOINTS_BASEDIR = "checkpoints" FRAMEWORK_BASEDIR = "framework" @@ -62,7 +63,7 @@ def set_framework_dir(self, dest): self.framework_dir = os.path.abspath(dest) def run(self, input_list): - tmp_folder = tempfile.mkdtemp(prefix="eos-") + tmp_folder = make_temp_dir(prefix="eos-") data_file = os.path.join(tmp_folder, self.DATA_FILE) output_file = os.path.join(tmp_folder, self.OUTPUT_FILE) log_file = os.path.join(tmp_folder, self.LOG_FILE) diff --git a/ersilia/hub/fetch/lazy_fetchers/dockerhub.py b/ersilia/hub/fetch/lazy_fetchers/dockerhub.py index 5307ae918..b3f98fa9a 100644 --- a/ersilia/hub/fetch/lazy_fetchers/dockerhub.py +++ b/ersilia/hub/fetch/lazy_fetchers/dockerhub.py @@ -106,7 +106,7 @@ def modify_information(self, model_id): :param service_class_file: File containing the model service class. :size_file: File containing the size of the pulled docker image. """ - information_file = "{0}/dest/{1}/{2}".format(EOS, model_id, INFORMATION_FILE) + information_file = os.path.join(self._model_path(model_id), INFORMATION_FILE) mp = ModelPuller(model_id=model_id, config_json=self.config_json) try: with open(information_file, "r") as infile: @@ -115,9 +115,11 @@ def modify_information(self, model_id): self.logger.error("Information file not found, not modifying anything") return None - data["service_class"] = "pulled_docker" # Using this literal here to prevent a file read - # from service class file for a model fetched through DockerHub since we already know the service class. - data["size"] = mp._get_size_of_local_docker_image_in_mb() + # Using this literal here to prevent a file read + # from service class file for a model fetched through DockerHub + # since we already know the service class. + data["service_class"] = "pulled_docker" + data["size"] = mp._get_size_of_local_docker_image_in_mb() # TODO this should probably be a util function with open(information_file, "w") as outfile: json.dump(data, outfile, indent=4) diff --git a/ersilia/hub/fetch/pack/bentoml_pack/runners.py b/ersilia/hub/fetch/pack/bentoml_pack/runners.py index d7418e1ce..5ff58a836 100644 --- a/ersilia/hub/fetch/pack/bentoml_pack/runners.py +++ b/ersilia/hub/fetch/pack/bentoml_pack/runners.py @@ -19,6 +19,7 @@ from ... import MODEL_INSTALL_COMMANDS_FILE from ..... import throw_ersilia_exception from .....utils.exceptions_utils.fetch_exceptions import CondaEnvironmentExistsError +from .....utils.logging import make_temp_dir USE_CHECKSUM = False @@ -199,7 +200,7 @@ def _run(self): self.logger.debug("Executing container {0}".format(name)) self.docker.exec_container(name, "python %s" % self.cfg.HUB.PACK_SCRIPT) self.logger.debug("Copying bundle from docker image to host") - tmp_dir = tempfile.mkdtemp(prefix="ersilia-") + tmp_dir = make_temp_dir(prefix="ersilia-") self.logger.debug("Using this temporary directory: {0}".format(tmp_dir)) self.docker.cp_from_container( name, "/root/bentoml/repository/%s" % model_id, tmp_dir diff --git a/ersilia/hub/fetch/register/standard_example.py b/ersilia/hub/fetch/register/standard_example.py index 13c73ba10..fffe20f10 100644 --- a/ersilia/hub/fetch/register/standard_example.py +++ b/ersilia/hub/fetch/register/standard_example.py @@ -46,9 +46,8 @@ def run(self): self.logger.debug(cmd_output) if "Welcome to Ersilia" in cmd_output: self.logger.debug("No need to use Conda!") - for cmd in commands: - self.logger.debug(cmd) - run_command(cmd) + cmd = " && ".join(commands) + run_command(cmd) else: self.logger.debug("Will run this through Conda") env_name = os.environ.get("CONDA_DEFAULT_ENV") diff --git a/ersilia/hub/pull/pull.py b/ersilia/hub/pull/pull.py index e809f4e73..ccb869f80 100644 --- a/ersilia/hub/pull/pull.py +++ b/ersilia/hub/pull/pull.py @@ -15,6 +15,7 @@ from ...utils.docker import SimpleDocker from ...default import DOCKERHUB_ORG, DOCKERHUB_LATEST_TAG, EOS, MODEL_SIZE_FILE +from ...utils.logging import make_temp_dir PULL_IMAGE = os.environ.get("PULL_IMAGE", "Y") @@ -112,7 +113,7 @@ def pull(self): "Trying to pull image {0}/{1}".format(DOCKERHUB_ORG, self.model_id) ) tmp_file = os.path.join( - tempfile.mkdtemp(prefix="ersilia-"), "docker_pull.log" + make_temp_dir(prefix="ersilia-"), "docker_pull.log" ) self.logger.debug("Keeping logs of pull in {0}".format(tmp_file)) run_command( diff --git a/ersilia/io/output.py b/ersilia/io/output.py index fde0b5f73..ca5350abf 100644 --- a/ersilia/io/output.py +++ b/ersilia/io/output.py @@ -15,6 +15,7 @@ from ..db.hubdata.json_models_interface import JsonModelsInterface from ..default import FEATURE_MERGE_PATTERN, PACK_METHOD_FASTAPI from ..utils.paths import resolve_pack_method +from ..utils.logging import make_temp_dir class DataFrame(object): @@ -437,7 +438,7 @@ def __init__(self, config_json): GenericOutputAdapter.__init__(self, config_json=config_json) def dictlist2dataframe(self, dl, model_id, api_name): - tmp_dir = tempfile.mkdtemp(prefix="ersilia-") + tmp_dir = make_temp_dir(prefix="ersilia-") df_file = os.path.join(tmp_dir, "data.csv") self.adapt(dl, df_file, model_id, api_name) df = Dataframe() diff --git a/ersilia/io/readers/file.py b/ersilia/io/readers/file.py index 6bc3156ea..b25aa0254 100644 --- a/ersilia/io/readers/file.py +++ b/ersilia/io/readers/file.py @@ -7,8 +7,8 @@ from ..shape import InputShape from ..shape import InputShapeSingle, InputShapeList, InputShapePairOfLists - from ... import logger +from ...utils.logging import make_temp_dir MIN_COLUMN_VALIDITY = 0.8 FLATTENED_EVIDENCE = 0.2 @@ -68,7 +68,7 @@ def get_extension(self): class BatchCacher(object): def __init__(self): - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + self.tmp_folder = make_temp_dir(prefix="ersilia-") def get_cached_files(self, prefix): idx2fn = {} @@ -554,7 +554,7 @@ def split_in_cache(self): class TabularFileReader(StandardTabularFileReader): def __init__(self, path, IO, sniff_line_limit=100): self.src_path = os.path.abspath(path) - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + self.tmp_folder = make_temp_dir(prefix="ersilia-") self.dst_path = os.path.join(self.tmp_folder, "standard_input_file.csv") self.path = self.dst_path self.IO = IO @@ -720,7 +720,7 @@ def split_in_cache(self): class JsonFileReader(StandardJsonFileReader): def __init__(self, path, IO): self.src_path = os.path.abspath(path) - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + self.tmp_folder = make_temp_dir(prefix="ersilia-") self.dst_path = os.path.join(self.tmp_folder, "standard_input_file.json") self.path = self.dst_path self.IO = IO diff --git a/ersilia/publish/metadata.py b/ersilia/publish/metadata.py index c59fec0b1..4c2ffe9d8 100644 --- a/ersilia/publish/metadata.py +++ b/ersilia/publish/metadata.py @@ -3,6 +3,7 @@ from ..hub.content.card import ReadmeMetadata, AirtableMetadata, RepoMetadataFile from ..utils.terminal import run_command +from ..utils.logging import make_temp_dir from .. import ErsiliaBase from ..default import GITHUB_ORG @@ -16,7 +17,7 @@ def __init__(self, model_id=None, repo_path=None, commit=True, config_json=None) else: self.repo_path = None self.commit = commit - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-os") + self.tmp_folder = make_temp_dir(prefix="ersilia-os") self.cwd = os.getcwd() ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) @@ -68,7 +69,7 @@ def update(self): class JsonUpdater(ErsiliaBase): def __init__(self, model_id, config_json=None): self.model_id = model_id - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-os") + self.tmp_folder = make_temp_dir(prefix="ersilia-os") self.cwd = os.getcwd() ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) diff --git a/ersilia/publish/s3.py b/ersilia/publish/s3.py index db12c3cee..852abb90d 100644 --- a/ersilia/publish/s3.py +++ b/ersilia/publish/s3.py @@ -5,6 +5,7 @@ import zipfile from ..utils.terminal import run_command +from ..utils.logging import make_temp_dir from .. import ErsiliaBase from ..default import ERSILIA_MODELS_S3_BUCKET, ERSILIA_MODELS_ZIP_S3_BUCKET @@ -16,8 +17,8 @@ def __init__(self, model_id, config_json=None): self.model_id = model_id ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) self.cwd = os.getcwd() - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") - self.tmp_zip_folder = tempfile.mkdtemp(prefix="ersilia-") + self.tmp_folder = make_temp_dir(prefix="ersilia-") + self.tmp_zip_folder = make_temp_dir(prefix="ersilia-") self.aws_access_key_id = None self.aws_secret_access_key = None self.ignore = ["upload_model_to_s3.py"] diff --git a/ersilia/publish/test.py b/ersilia/publish/test.py index 31dfd3592..db2ca9cfd 100644 --- a/ersilia/publish/test.py +++ b/ersilia/publish/test.py @@ -15,6 +15,7 @@ from .. import throw_ersilia_exception from .. import ErsiliaModel from ..utils.exceptions_utils import test_exceptions as texc +from ..utils.logging import make_temp_dir from ..utils.terminal import run_command_check_output from ..core.session import Session from ..default import INFORMATION_FILE @@ -39,7 +40,7 @@ def __init__(self, model_id, config_json=None): ErsiliaBase.__init__(self, config_json=config_json, credentials_json=None) self.model_id = model_id self.model_size = 0 - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + self.tmp_folder = make_temp_dir(prefix="ersilia-") self._info = self._read_information() self._input = self._info["card"]["Input"] self._output_type = self._info["card"]["Output Type"] diff --git a/ersilia/serve/api.py b/ersilia/serve/api.py index 8f83900d8..b28a1fecb 100644 --- a/ersilia/serve/api.py +++ b/ersilia/serve/api.py @@ -14,6 +14,7 @@ from .schema import ApiSchema from ..utils.exceptions_utils.api_exceptions import InputFileNotFoundError +from ..utils.logging import make_temp_dir class Api(object): @@ -122,7 +123,7 @@ def _post(self, input, output): def post_only_calculations(self, input, output, batch_size): self._batch_size = batch_size if output is not None: - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") fmt = output.split(".")[-1] output_base = ".".join(os.path.basename(output).split(".")[:-1]) i = 0 @@ -151,7 +152,7 @@ def _post_reads(self, input, output): def post_only_reads(self, input, output, batch_size): self._batch_size = batch_size if output is not None: - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") fmt = output.split(".")[-1] output_base = ".".join(os.path.basename(output).split(".")[:-1]) i = 0 @@ -222,7 +223,7 @@ def post_amenable_to_h5(self, input, output, batch_size): self.logger.debug( "Checking for already available calculations in the data lake" ) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") done_input = os.path.join(tmp_folder, "done_input.csv") todo_input = os.path.join(tmp_folder, "todo_input.csv") cur_idx = 0 diff --git a/ersilia/serve/services.py b/ersilia/serve/services.py index 33f6c3d3b..0e3f56d4d 100644 --- a/ersilia/serve/services.py +++ b/ersilia/serve/services.py @@ -18,14 +18,16 @@ from ..utils.venv import SimpleVenv from ..default import DEFAULT_VENV from ..default import PACKMODE_FILE, APIS_LIST_FILE -from ..default import DOCKERHUB_ORG, DOCKERHUB_LATEST_TAG +from ..default import DOCKERHUB_ORG, DOCKERHUB_LATEST_TAG, CONTAINER_LOGS_TMP_DIR from ..default import IS_FETCHED_FROM_HOSTED_FILE from ..default import INFORMATION_FILE from ..default import PACK_METHOD_BENTOML, PACK_METHOD_FASTAPI +from ..utils.session import get_session_dir from ..utils.exceptions_utils.serve_exceptions import ( BadGatewayError, DockerNotActiveError, ) +from ..utils.logging import make_temp_dir from ..setup.requirements.conda import CondaRequirement SLEEP_SECONDS = 1 @@ -57,7 +59,7 @@ def _get_apis_from_apis_list(self): def _get_info_from_bento(self): """Get info available from the Bento""" - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "information.json") cmd = "bentoml info --quiet {0}:{1} > {2}".format( self.model_id, self.bundle_tag, tmp_file @@ -139,7 +141,7 @@ def serve(self, runcommand_func=None): ) ) self.logger.debug("Free port: {0}".format(self.port)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "serve.sh") tmp_file = os.path.join(tmp_folder, "serve.log") tmp_pid = os.path.join(tmp_folder, "serve.pid") @@ -246,7 +248,7 @@ def serve(self, runcommand_func=None): ) ) self.logger.debug("Free port: {0}".format(self.port)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "serve.sh") tmp_file = os.path.join(tmp_folder, "serve.log") tmp_pid = os.path.join(tmp_folder, "serve.pid") @@ -653,12 +655,14 @@ def __init__(self, model_id, config_json=None, preferred_port=None, url=None): DOCKERHUB_ORG, self.model_id, DOCKERHUB_LATEST_TAG ) self.logger.debug("Starting Docker Daemon service") - self.tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + self.container_tmp_logs = os.path.join(get_session_dir(), CONTAINER_LOGS_TMP_DIR) self.logger.debug( - "Creating temporary folder {0} and mounting as volume in container".format( - self.tmp_folder + "Creating container tmp logs folder {0} and mounting as volume in container".format( + self.container_tmp_logs ) ) + if not os.path.exists(self.container_tmp_logs): + os.makedirs(self.container_tmp_logs) self.simple_docker = SimpleDocker() self.pid = -1 self._mem_gb = self._get_memory() @@ -778,7 +782,7 @@ def _wait_until_container_is_running(self): def serve(self): self._stop_all_containers_of_image() self.container_name = "{0}_{1}".format(self.model_id, str(uuid.uuid4())[:4]) - self.volumes = {self.tmp_folder: {"bind": "/ersilia_tmp", "mode": "rw"}} + self.volumes = {self.container_tmp_logs: {"bind": "/tmp", "mode": "rw"}} self.logger.debug("Trying to run container") if self._mem_gb is None: self.container = self.client.containers.run( diff --git a/ersilia/setup/baseconda.py b/ersilia/setup/baseconda.py index 552a7eff4..78e5a0cb3 100644 --- a/ersilia/setup/baseconda.py +++ b/ersilia/setup/baseconda.py @@ -6,6 +6,7 @@ from ..utils.terminal import run_command from ..utils.versioning import Versioner from .utils.clone import ErsiliaCloner +from ..utils.logging import make_temp_dir from .. import logger @@ -58,7 +59,7 @@ def _get_env_name(self, org, tag): return self.versions.base_conda_name(org, tag) def find_closest_python_version(self, python_version): - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "conda_search_python.txt") tmp_script = os.path.join(tmp_folder, "script.sh") is_base = self.conda.is_base() @@ -96,7 +97,7 @@ def setup(self, org, tag): return ptag = self._parse_tag(tag) cmd = self._install_command(org, tag) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") if self._is_ersiliaos(org): tmp_repo = self.cloner.clone(tmp_folder, version=ptag["ver"]) else: diff --git a/ersilia/setup/basedocker.py b/ersilia/setup/basedocker.py index 9d49277e9..2ad0c50bc 100644 --- a/ersilia/setup/basedocker.py +++ b/ersilia/setup/basedocker.py @@ -4,6 +4,7 @@ from ..utils.versioning import Versioner from .. import ErsiliaBase from .utils.clone import ErsiliaCloner +from ..utils.logging import make_temp_dir # TODO: Make sure it is used. @@ -39,7 +40,7 @@ def setup(self, org, tag): return ptag = self._parse_tag(tag) # get a copy of the repository in a temporary directory - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_repo = self.cloner.clone(tmp_folder, version=ptag["ver"]) # write the dockerfile dockerfile = """ diff --git a/ersilia/setup/requirements/bentoml.py b/ersilia/setup/requirements/bentoml.py index b17fb30d7..951dbfc3f 100644 --- a/ersilia/setup/requirements/bentoml.py +++ b/ersilia/setup/requirements/bentoml.py @@ -21,7 +21,7 @@ def is_bentoml_ersilia_version(self): if not self.is_installed(): return False - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "version.txt") cmd = "bentoml --version > {0}".format(tmp_file) subprocess.Popen(cmd, shell=True).wait() diff --git a/ersilia/utils/__init__.py b/ersilia/utils/__init__.py index 0ea170c48..e9e8c2c1b 100644 --- a/ersilia/utils/__init__.py +++ b/ersilia/utils/__init__.py @@ -1,7 +1,9 @@ import os -from ..default import EOS +from ..default import SESSIONS_DIR +from ..utils.session import get_session_dir def tmp_pid_file(model_id): """Gets filename to store process ID""" - return os.path.join(EOS, "tmp", "{0}.pid".format(model_id)) + session_dir = get_session_dir() + return os.path.join(session_dir, "{0}.pid".format(model_id)) diff --git a/ersilia/utils/conda.py b/ersilia/utils/conda.py index b0730ed88..646f846bc 100644 --- a/ersilia/utils/conda.py +++ b/ersilia/utils/conda.py @@ -11,6 +11,7 @@ from ..default import CONDA_ENV_YML_FILE from .. import logger from ..utils.exceptions_utils.fetch_exceptions import ModelPackageInstallError +from ..utils.logging import make_temp_dir from .. import throw_ersilia_exception BASE = "base" @@ -235,7 +236,7 @@ def __init__(self, config_json=None): CondaUtils.__init__(self, config_json=config_json) def _env_list(self): - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "env_list.tsv") tmp_script = os.path.join(tmp_folder, "script.sh") bash_script = """ @@ -281,7 +282,7 @@ def startswith(self, environment): return envs_list def get_python_path_env(self, environment): - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_folder, "tmp.txt") self.run_commandlines(environment, "which python > {0}".format(tmp_file)) with open(tmp_file, "r") as f: @@ -291,7 +292,7 @@ def get_python_path_env(self, environment): def delete_one(self, environment): if not self.exists(environment): return - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") bash_script = self.activate_base() bash_script += """ @@ -327,7 +328,7 @@ def export_env_yml(self, environment, dest): if self.is_base(): return yml_file = os.path.join(dest, CONDA_ENV_YML_FILE) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") bash_script = self.activate_base() bash_script += """ @@ -350,7 +351,7 @@ def clone(self, src_env, dst_env): raise Exception("{0} source environment does not exist".format(src_env)) if self.exists(dst_env): raise Exception("{0} destination environment exists".format(dst_env)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") bash_script = self.activate_base() bash_script += """ @@ -398,12 +399,12 @@ def run_commandlines(self, environment, commandlines): logger.debug(commandlines) if not self.exists(environment): raise Exception("{0} environment does not exist".format(environment)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") logger.debug("Activating base environment") logger.debug("Current working directory: {0}".format(os.getcwd())) self.create_executable_bash_script(environment, commandlines, tmp_script) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_log = os.path.join(tmp_folder, "command_outputs.log") cmd = "bash {0} 2>&1 | tee -a {1}".format(tmp_script, tmp_log) logger.debug("Running {0}".format(cmd)) @@ -436,7 +437,7 @@ def run_commandlines(self, environment, commandlines): if not self.exists(environment): raise Exception("{0} environment does not exist".format(environment)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") logger.debug("Activating environment") logger.debug("Current working directory: {0}".format(os.getcwd())) @@ -449,7 +450,7 @@ def run_commandlines(self, environment, commandlines): with open(tmp_script, "w") as f: f.write(bash_script) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_log = os.path.join(tmp_folder, "command_outputs.log") cmd = "bash {0} 2>&1 | tee -a {1}".format(tmp_script, tmp_log) logger.debug("Running {0}".format(cmd)) diff --git a/ersilia/utils/docker.py b/ersilia/utils/docker.py index bd3b56c46..282495436 100644 --- a/ersilia/utils/docker.py +++ b/ersilia/utils/docker.py @@ -10,6 +10,7 @@ from .. import logger from ..default import DEFAULT_DOCKER_PLATFORM, DEFAULT_UDOCKER_USERNAME from ..utils.system import SystemChecker +from ..utils.logging import make_temp_dir def resolve_platform(): @@ -77,7 +78,7 @@ def _image_name(org, img, tag): return "%s/%s:%s" % (org, img, tag) def images(self): - tmp_dir = tempfile.mkdtemp(prefix="ersilia-") + tmp_dir = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_dir, "images.txt") if not self._with_udocker: cmd = "docker images > {0}".format(tmp_file) @@ -109,7 +110,7 @@ def images(self): return img_dict def containers(self, only_run): - tmp_dir = tempfile.mkdtemp(prefix="ersilia-") + tmp_dir = make_temp_dir(prefix="ersilia-") tmp_file = os.path.join(tmp_dir, "containers.txt") if not only_run: all_str = "-a" @@ -145,7 +146,7 @@ def exists(self, org, img, tag): echo "False" fi """ - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "exists.sh") with open(tmp_script, "w") as f: f.write(bash_script) @@ -214,7 +215,7 @@ def remove(name): @staticmethod def cp_from_container(name, img_path, local_path, org=None, img=None, tag=None): local_path = os.path.abspath(local_path) - tmp_file = os.path.join(tempfile.mkdtemp(prefix="ersilia-"), "tmp.txt") + tmp_file = os.path.join(make_temp_dir(prefix="ersilia-"), "tmp.txt") cmd = "docker cp %s:%s %s &> %s" % (name, img_path, local_path, tmp_file) run_command(cmd) with open(tmp_file, "r") as f: diff --git a/ersilia/utils/download.py b/ersilia/utils/download.py index 7b60175b6..dbb665129 100644 --- a/ersilia/utils/download.py +++ b/ersilia/utils/download.py @@ -14,7 +14,7 @@ from .. import logger from ..default import S3_BUCKET_URL, S3_BUCKET_URL_ZIP - +from ..utils.logging import make_temp_dir class PseudoDownloader(object): def __init__(self, overwrite): @@ -125,7 +125,7 @@ def _exists(self, destination): return False def _clone_with_git(self, org, repo, destination): - tmp_folder = os.path.abspath(tempfile.mkdtemp(prefix="ersilia-")) + tmp_folder = os.path.abspath(make_temp_dir(prefix="ersilia-")) script = """ #Disabling automatic LFS clone (s3 feature) by adding GIT_LFS_SKIP_SMUDGE=1 cd {0} @@ -136,7 +136,7 @@ def _clone_with_git(self, org, repo, destination): tmp_folder, org, repo, destination ) run_file = os.path.join( - os.path.abspath(tempfile.mkdtemp(prefix="ersilia")), "run.sh" + os.path.abspath(make_temp_dir(prefix="ersilia")), "run.sh" ) with open(run_file, "w") as f: f.write(script) @@ -225,7 +225,7 @@ def _git_lfs(self, destination, filename): # not downloaded from an S3 bucket or have unexpected sha256 value. self.logger.debug("⏳ Trying LFS clone for file {0}".format(filename)) script = "cd {0}; git lfs pull --include {1}".format(destination, filename) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") run_file = os.path.join(tmp_folder, "run_lfs.sh") with open(run_file, "w") as f: f.write(script) diff --git a/ersilia/utils/exceptions_utils/exceptions.py b/ersilia/utils/exceptions_utils/exceptions.py index a2111d523..c731a8792 100644 --- a/ersilia/utils/exceptions_utils/exceptions.py +++ b/ersilia/utils/exceptions_utils/exceptions.py @@ -2,6 +2,7 @@ import os import tempfile from ...utils.terminal import run_command +from ...utils.logging import make_temp_dir class ErsiliaError(Exception): @@ -90,7 +91,7 @@ def run_from_terminal(self): exec_file = os.path.join(framework_dir, exec_file) input_file = os.path.join(framework_dir, "example_input.csv") output_file = os.path.join(framework_dir, "example_output.csv") - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") log_file = os.path.join(tmp_folder, "terminal.log") run_command("ersilia example {0} -n 3 -f {1}".format(self.model_id, input_file)) cmd = "bash {0} {1} {2} {3} 2>&1 | tee -a {4}".format( diff --git a/ersilia/utils/installers.py b/ersilia/utils/installers.py index 450027a60..fa7dfdaa3 100644 --- a/ersilia/utils/installers.py +++ b/ersilia/utils/installers.py @@ -11,6 +11,7 @@ from .terminal import run_command from .versioning import Versioner import click +from .logging import make_temp_dir INSTALL_LOG_FILE = ".install.log" @@ -156,7 +157,7 @@ def base_conda(self): sc = SimpleConda() if sc.exists(eos_base_env): return - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_repo = self._clone_repo(tmp_folder) tmp_script = os.path.join(tmp_folder, "script.sh") tmp_python_script = os.path.join(tmp_folder, "base_installer.py") @@ -217,7 +218,7 @@ def server_docker(self): if docker.exists(org, img, tag): return # get a copy of the repository in a temporary directory - tmp_dir = tempfile.mkdtemp(prefix="ersilia-") + tmp_dir = make_temp_dir(prefix="ersilia-") tmp_repo = self._clone_repo(tmp_dir) # write the dockerfile dockerfile = """ diff --git a/ersilia/utils/logging.py b/ersilia/utils/logging.py index c09249875..fd6a6e34a 100644 --- a/ersilia/utils/logging.py +++ b/ersilia/utils/logging.py @@ -1,13 +1,28 @@ -import sys import os +import sys import json +from pathlib import Path +import tempfile from loguru import logger -from ..default import EOS, LOGGING_FILE, CURRENT_LOGGING_FILE, VERBOSE_FILE +from ..default import LOGGING_FILE, CURRENT_LOGGING_FILE, VERBOSE_FILE +from ..utils.session import get_session_dir ROTATION = "10 MB" +def make_temp_dir(prefix): + tmp_dir = tempfile.mkdtemp(prefix=prefix) + tmp_dirname = os.path.basename(tmp_dir) + logs_tmp_dir = os.path.join(get_session_dir(), "logs", "tmp") + if not os.path.exists(logs_tmp_dir): + os.makedirs(logs_tmp_dir) + dst = Path(os.path.join(logs_tmp_dir, tmp_dirname)) + src = Path(tmp_dir) + dst.symlink_to(src, target_is_directory=True) + return tmp_dir + + class Logger(object): def __init__(self): self.logger = logger @@ -15,24 +30,23 @@ def __init__(self): self._console = None self._file = None self.fmt = "{time:HH:mm:ss} | {level: <8} | {message}" - self._verbose_file = os.path.join(EOS, VERBOSE_FILE) + self._verbose_file = os.path.join(get_session_dir(), VERBOSE_FILE) self._log_to_console() self._log_to_file() self._log_to_current_file() self._log_terminal_commands_to_console() def _log_to_file(self): + logging_file = os.path.join(get_session_dir(), LOGGING_FILE) self._file = self.logger.add( - os.path.join(EOS, LOGGING_FILE), format=self.fmt, rotation=ROTATION + logging_file, format=self.fmt, rotation=ROTATION ) def _log_to_current_file(self): - current_log_file = os.path.join(EOS, CURRENT_LOGGING_FILE) - - if os.path.exists(current_log_file): - os.remove(current_log_file) + session_dir = get_session_dir() + current_log_file = os.path.join(session_dir, CURRENT_LOGGING_FILE) self._current_file = self.logger.add( - os.path.join(EOS, CURRENT_LOGGING_FILE), format=self.fmt, rotation=ROTATION + current_log_file, format=self.fmt, rotation=ROTATION ) def _log_to_console(self): diff --git a/ersilia/utils/session.py b/ersilia/utils/session.py new file mode 100644 index 000000000..70a7b4bce --- /dev/null +++ b/ersilia/utils/session.py @@ -0,0 +1,54 @@ +import os +import shutil +import psutil + +from ..default import SESSIONS_DIR, LOGS_DIR, CONTAINER_LOGS_TMP_DIR + +def get_current_pid(): + return os.getpid() + +def get_parent_pid(): + pid = os.getppid() + return pid + +def create_session_files(session_name): + session_dir = os.path.join(SESSIONS_DIR, session_name) + os.makedirs(os.path.join(session_dir, LOGS_DIR), exist_ok=True) + os.makedirs(os.path.join(session_dir, CONTAINER_LOGS_TMP_DIR), exist_ok=True) + +def create_session_dir(): + remove_orphaned_sessions() + session_name = f"session_{get_parent_pid()}" + session_dir = os.path.join(SESSIONS_DIR, session_name) + os.makedirs(session_dir, exist_ok=True) + create_session_files(session_name) + +def get_session_dir(): + return os.path.join(SESSIONS_DIR, get_session_id()) + +def remove_session_dir(session_name): + session_dir = os.path.join(SESSIONS_DIR, session_name) + shutil.rmtree(session_dir) + +def determine_orphaned_session(): + # TODO maybe this is slow, look out for performance + _sessions = [] + sessions = list(filter(lambda s: s.startswith("session_"), os.listdir(SESSIONS_DIR))) + if sessions: + for session in sessions: + session_pid = int(session.split("_")[1]) + if not psutil.pid_exists(session_pid): + _sessions.append(session) + return _sessions + +def remove_orphaned_sessions(): + orphaned_sessions = determine_orphaned_session() + for session in orphaned_sessions: + try: + remove_session_dir(session) + except PermissionError: #TODO Is there a better way to handle this? + pass # TODO we should at least log this somehow + +def get_session_id(): + return f"session_{get_parent_pid()}" + diff --git a/ersilia/utils/terminal.py b/ersilia/utils/terminal.py index f78c7aa2d..5c57eddde 100644 --- a/ersilia/utils/terminal.py +++ b/ersilia/utils/terminal.py @@ -10,11 +10,13 @@ inputimeout = None TimeoutOccurred = None -from ..default import EOS, VERBOSE_FILE +from ..default import VERBOSE_FILE +from ..utils.session import get_session_dir +from ..utils.logging import make_temp_dir def is_quiet(): - verbose_file = os.path.join(EOS, VERBOSE_FILE) + verbose_file = os.path.join(get_session_dir(), VERBOSE_FILE) if not os.path.exists(verbose_file): return False with open(verbose_file, "r") as f: @@ -51,7 +53,8 @@ def run_command(cmd, quiet=None): def run_command_check_output(cmd): if type(cmd) is str: assert ">" not in cmd - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") + os.listdir(tmp_folder) tmp_file = os.path.join(tmp_folder, "out.txt") cmd = cmd + " > " + tmp_file with open(os.devnull, "w") as fp: diff --git a/ersilia/utils/venv.py b/ersilia/utils/venv.py index 616a1aaea..ae6487ca6 100644 --- a/ersilia/utils/venv.py +++ b/ersilia/utils/venv.py @@ -11,6 +11,7 @@ ) from .. import throw_ersilia_exception from .. import logger +from ..utils.logging import make_temp_dir class SimpleVenv(ErsiliaBase): @@ -47,7 +48,7 @@ def delete(self, environment): def run_commandlines(self, environment, commandlines): if not self.exists(environment): raise Exception("{0} environment does not exist".format(environment)) - tmp_folder = tempfile.mkdtemp(prefix="ersilia-") + tmp_folder = make_temp_dir(prefix="ersilia-") tmp_script = os.path.join(tmp_folder, "script.sh") tmp_log = os.path.join(tmp_folder, "installs.log") # new with open(tmp_script, "w") as f: