adding file changes to accomodate separate upload log directory

iridauploader/config/config.py

@@ -51,6 +51,7 @@ def _init_config_parser():
                         SettingsDefault._make(["minimum_file_size", 0]),  # default minimum file size in kb
                         SettingsDefault._make(["http_max_retries", 5]),
                         SettingsDefault._make(["http_backoff_factor", 0]),
+                        SettingsDefault._make(["log_directory", None]),
                         ]
     # add defaults to config parser
     for config in default_settings:
@@ -134,7 +135,8 @@ def set_config_options(client_id=None,
                        timeout=None,
                        minimum_file_size=None,
                        http_max_retries=None,
-                       http_backoff_factor=None):
+                       http_backoff_factor=None,
+                       log_directory=None):
     """
     Updates the config options for all not None parameters
     :param client_id:
@@ -149,6 +151,7 @@ def set_config_options(client_id=None,
     :param minimum_file_size:
     :param http_max_retries:
     :param http_backoff_factor:
+    :param log_directory:
     :return:
     """
     global _conf_parser
@@ -198,6 +201,10 @@ def set_config_options(client_id=None,
         # http_backoff_factor is always a float
         logging.debug("Setting 'http_backoff_factor' config to {}".format(http_backoff_factor))
         _update_config_option('http_backoff_factor', http_backoff_factor)
+    if log_directory is not None:
+        # log_directory is always a str
+        logging.debug("Setting 'log_directory' config to {}".format(log_directory))
+        _update_config_option('log_directory', log_directory)
 
 
 def setup():

iridauploader/core/upload.py

@@ -13,7 +13,7 @@
 import iridauploader.parsers as parsers
 import iridauploader.progress as progress
 from iridauploader.model import DirectoryStatus
-
+import os
 from . import api_handler, parsing_handler, logger, exit_return, upload_helpers
 
 
@@ -271,8 +271,18 @@ def logging_start_block(directory):
     Includes the uploader version number set in this module
     :return:
     """
-    if config.read_config_option("readonly", bool, False) is False:
-        logger.add_log_to_directory(directory)
+    log_directory = config.read_config_option("log_directory")
+    #print(log_directory)
+    if log_directory != None:
+        run_name = os.path.basename(directory)
+        log_directory_run_path = os.path.join(log_directory, run_name)
+        #print(log_directory_run_path)
+        if not os.path.isdir(log_directory_run_path):
+            os.mkdir(log_directory_run_path)
+    else:
+        log_directory_run_path = directory
+    if config.read_config_option("readonly", bool, False) is False or log_directory != None:
+        logger.add_log_to_directory(log_directory_run_path)
     logging.info("==================================================")
     logging.info("---------------STARTING UPLOAD RUN----------------")
     logging.info("Uploader Version {}".format(VERSION_NUMBER))

iridauploader/progress/upload_status.py

@@ -28,7 +28,8 @@ def get_directory_status(directory, required_file_list):
     :return: directory and status dictionary
     """
     # Verify directory is readable
-    if not os.access(directory, os.R_OK):
+    log_directory = config.read_config_option("log_directory")
+    if not os.access(directory, os.R_OK) and log_directory == None:
         return DirectoryStatus(directory=directory,
                                status=DirectoryStatus.INVALID,
                                message='Directory cannot be read. Please check permissions')
@@ -54,8 +55,12 @@ def get_directory_status(directory, required_file_list):
 
     # All pre-validation passed
     # Determine if status file already exists, or if the run is brand new
-    if STATUS_FILE_NAME in file_list:  # Status file already exists, use it.
-        return read_directory_status_from_file(directory)
+    if log_directory != None:
+        status_directory = os.path.join(log_directory, directory)
+    else:
+        status_directory = directory
+    if os.path.isfile(os.path.join(status_directory, STATUS_FILE_NAME)):  # Status file already exists, use it.
+        return read_directory_status_from_file(status_directory)
     else:  # no irida_uploader_status.info file yet, has not been uploaded
         return DirectoryStatus(directory=directory, status=DirectoryStatus.NEW)
 
@@ -96,13 +101,20 @@ def write_directory_status(directory_status):
     :param directory_status: DirectoryStatus object containing status to write to directory
     :return: None
     """
-    if config.read_config_option("readonly", bool, False) is False:
-        if not os.access(directory_status.directory, os.W_OK):  # Cannot access upload directory
+    log_directory = config.read_config_option("log_directory")
+    if config.read_config_option("readonly", bool, False) is False or log_directory != None:
+        if not os.access(directory_status.directory, os.W_OK) and log_directory == None:  # check if directory can be accessed, or that the log_directory is set
             raise exceptions.DirectoryError("Cannot access directory", directory_status.directory)
-
+        elif not os.access(log_directory, os.W_OK):
+            raise exceptions.DirectoryError("log directory set but no write access")
         json_data = directory_status.to_json_dict()
-
-        uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME)
+        if log_directory != None:
+            run_log_path = os.path.join(log_directory, os.path.basename(directory_status.directory))
+            if not os.path.isdir(run_log_path):
+                os.mkdir(run_log_path)
+            uploader_info_file = os.path.join(run_log_path, STATUS_FILE_NAME)
+        else:
+            uploader_info_file = os.path.join(directory_status.directory, STATUS_FILE_NAME)
         with open(uploader_info_file, "w") as json_file:
             json.dump(json_data, json_file, indent=4, sort_keys=True)
             json_file.write("\n")