From 158d40df33c1873456320cd49f84f24ac21d43b0 Mon Sep 17 00:00:00 2001 From: chrlar Date: Mon, 18 Nov 2024 15:16:27 +0100 Subject: [PATCH] adding file changes to accomodate separate upload log directory --- iridauploader/config/config.py | 9 +++++++- iridauploader/core/upload.py | 16 +++++++++++--- iridauploader/progress/upload_status.py | 28 ++++++++++++++++++------- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/iridauploader/config/config.py b/iridauploader/config/config.py index 9736c01..cf17fa7 100644 --- a/iridauploader/config/config.py +++ b/iridauploader/config/config.py @@ -51,6 +51,7 @@ def _init_config_parser(): SettingsDefault._make(["minimum_file_size", 0]), # default minimum file size in kb SettingsDefault._make(["http_max_retries", 5]), SettingsDefault._make(["http_backoff_factor", 0]), + SettingsDefault._make(["log_directory", None]), ] # add defaults to config parser for config in default_settings: @@ -134,7 +135,8 @@ def set_config_options(client_id=None, timeout=None, minimum_file_size=None, http_max_retries=None, - http_backoff_factor=None): + http_backoff_factor=None, + log_directory=None): """ Updates the config options for all not None parameters :param client_id: @@ -149,6 +151,7 @@ def set_config_options(client_id=None, :param minimum_file_size: :param http_max_retries: :param http_backoff_factor: + :param log_directory: :return: """ global _conf_parser @@ -198,6 +201,10 @@ def set_config_options(client_id=None, # http_backoff_factor is always a float logging.debug("Setting 'http_backoff_factor' config to {}".format(http_backoff_factor)) _update_config_option('http_backoff_factor', http_backoff_factor) + if log_directory is not None: + # log_directory is always a str + logging.debug("Setting 'log_directory' config to {}".format(log_directory)) + _update_config_option('log_directory', log_directory) def setup(): diff --git a/iridauploader/core/upload.py b/iridauploader/core/upload.py index f402747..ed4401d 100644 --- a/iridauploader/core/upload.py +++ b/iridauploader/core/upload.py @@ -13,7 +13,7 @@ import iridauploader.parsers as parsers import iridauploader.progress as progress from iridauploader.model import DirectoryStatus - +import os from . import api_handler, parsing_handler, logger, exit_return, upload_helpers @@ -271,8 +271,18 @@ def logging_start_block(directory): Includes the uploader version number set in this module :return: """ - if config.read_config_option("readonly", bool, False) is False: - logger.add_log_to_directory(directory) + log_directory = config.read_config_option("log_directory") + #print(log_directory) + if log_directory != None: + run_name = os.path.basename(directory) + log_directory_run_path = os.path.join(log_directory, run_name) + #print(log_directory_run_path) + if not os.path.isdir(log_directory_run_path): + os.mkdir(log_directory_run_path) + else: + log_directory_run_path = directory + if config.read_config_option("readonly", bool, False) is False or log_directory != None: + logger.add_log_to_directory(log_directory_run_path) logging.info("==================================================") logging.info("---------------STARTING UPLOAD RUN----------------") logging.info("Uploader Version {}".format(VERSION_NUMBER)) diff --git a/iridauploader/progress/upload_status.py b/iridauploader/progress/upload_status.py index 692a4d7..4d39e34 100644 --- a/iridauploader/progress/upload_status.py +++ b/iridauploader/progress/upload_status.py @@ -28,7 +28,8 @@ def get_directory_status(directory, required_file_list): :return: directory and status dictionary """ # Verify directory is readable - if not os.access(directory, os.R_OK): + log_directory = config.read_config_option("log_directory") + if not os.access(directory, os.R_OK) and log_directory == None: return DirectoryStatus(directory=directory, status=DirectoryStatus.INVALID, message='Directory cannot be read. Please check permissions') @@ -54,8 +55,12 @@ def get_directory_status(directory, required_file_list): # All pre-validation passed # Determine if status file already exists, or if the run is brand new - if STATUS_FILE_NAME in file_list: # Status file already exists, use it. - return read_directory_status_from_file(directory) + if log_directory != None: + status_directory = os.path.join(log_directory, directory) + else: + status_directory = directory + if os.path.isfile(os.path.join(status_directory, STATUS_FILE_NAME)): # Status file already exists, use it. + return read_directory_status_from_file(status_directory) else: # no irida_uploader_status.info file yet, has not been uploaded return DirectoryStatus(directory=directory, status=DirectoryStatus.NEW) @@ -96,13 +101,20 @@ def write_directory_status(directory_status): :param directory_status: DirectoryStatus object containing status to write to directory :return: None """ - if config.read_config_option("readonly", bool, False) is False: - if not os.access(directory_status.directory, os.W_OK): # Cannot access upload directory + log_directory = config.read_config_option("log_directory") + if config.read_config_option("readonly", bool, False) is False or log_directory != None: + if not os.access(directory_status.directory, os.W_OK) and log_directory == None: # check if directory can be accessed, or that the log_directory is set raise exceptions.DirectoryError("Cannot access directory", directory_status.directory) - + elif not os.access(log_directory, os.W_OK): + raise exceptions.DirectoryError("log directory set but no write access") json_data = directory_status.to_json_dict() - - uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME) + if log_directory != None: + run_log_path = os.path.join(log_directory, os.path.basename(directory_status.directory)) + if not os.path.isdir(run_log_path): + os.mkdir(run_log_path) + uploader_info_file = os.path.join(run_log_path, STATUS_FILE_NAME) + else: + uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME) with open(uploader_info_file, "w") as json_file: json.dump(json_data, json_file, indent=4, sort_keys=True) json_file.write("\n")