diff --git "a/pages/3_\360\237\223\226_TOPP_Workflow_Framework.py" "b/pages/3_\360\237\223\226_TOPP_Workflow_Framework.py" index b2ef5e0..677f02c 100644 --- "a/pages/3_\360\237\223\226_TOPP_Workflow_Framework.py" +++ "b/pages/3_\360\237\223\226_TOPP_Workflow_Framework.py" @@ -23,24 +23,9 @@ - workflow output updates automatically in short intervalls - user can leave the app and return to the running workflow at any time - quickly build a workflow with multiple steps channelling files between steps -# """ ) -with st.expander("**Example User Interface**", True): - t = st.tabs(["📁 **File Upload**", "⚙️ **Configure**", "🚀 **Run**", "📊 **Results**"]) - with t[0]: - wf.show_file_upload_section() - - with t[1]: - wf.show_parameter_section() - - with t[2]: - wf.show_execution_section() - - with t[3]: - wf.show_results_section() - st.markdown( """ ## Quickstart @@ -117,26 +102,26 @@ **3. Choose `self.ui.input_python` to automatically generate complete input sections for a custom Python tool:** Takes the obligatory **script_file** argument. The default location for the Python script files is in `src/python-tools` (in this case the `.py` file extension is optional in the **script_file** argument), however, any other path can be specified as well. Parameters need to be specified in the Python script in the **DEFAULTS** variable with the mandatory **key** and **value** parameters. - -Here are the options to use as dictionary keys for parameter definitions (see `src/python-tools/example.py` for an example): - -Mandatory keys for each parameter -- **key:** a unique identifier -- **value:** the default value - -Optional keys for each parameter -- **name:** the name of the parameter -- **hide:** don't show the parameter in the parameter section (e.g. for **input/output files**) -- **options:** a list of valid options for the parameter -- **min:** the minimum value for the parameter (int and float) -- **max:** the maximum value for the parameter (int and float) -- **step_size:** the step size for the parameter (int and float) -- **help:** a description of the parameter -- **widget_type:** the type of widget to use for the parameter (default: auto) -- **advanced:** whether or not the parameter is advanced (default: False) - """) +with st.expander("Options to use as dictionary keys for parameter definitions (see `src/python-tools/example.py` for an example)"): + st.markdown(""" +**Mandatory** keys for each parameter +- *key:* a unique identifier +- *value:* the default value + +**Optional** keys for each parameter +- *name:* the name of the parameter +- *hide:* don't show the parameter in the parameter section (e.g. for **input/output files**) +- *options:* a list of valid options for the parameter +- *min:* the minimum value for the parameter (int and float) +- *max:* the maximum value for the parameter (int and float) +- *step_size:* the step size for the parameter (int and float) +- *help:* a description of the parameter +- *widget_type:* the type of widget to use for the parameter (default: auto) +- *advanced:* whether or not the parameter is advanced (default: False) +""") + st.code( getsource(Workflow.configure) ) @@ -201,7 +186,7 @@ """) st.code(""" -self.executor.run_command(["command", "arg1", "arg2", ...], write_log=True) +self.executor.run_command(["command", "arg1", "arg2", ...]) """) st.markdown( @@ -266,134 +251,4 @@ st.help(CommandExecutor.run_command) st.help(CommandExecutor.run_multiple_commands) st.help(CommandExecutor.run_topp) - st.help(CommandExecutor.run_python) - -with st.expander("**Example output of the complete example workflow**"): - st.code(""" -STARTING WORKFLOW - -Number of input mzML files: 2 - -Running 2 commands in parallel... - -Running command: -FeatureFinderMetabo -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML -out ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML -algorithm:common:chrom_peak_snr 4.0 -algorithm:common:noise_threshold_int 1000.0 -Waiting for command to finish... - -Running command: -FeatureFinderMetabo -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML -out ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML -algorithm:common:chrom_peak_snr 4.0 -algorithm:common:noise_threshold_int 1000.0 -Waiting for command to finish... - -Process finished: -FeatureFinderMetabo -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML -out ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML -algorithm:common:chrom_peak_snr 4.0 -algorithm:common:noise_threshold_int 1000.0 -Total time to run command: 0.55 seconds - -Progress of 'loading mzML': - Progress of 'loading spectra list': - - 89.06 % - -- done [took 0.17 s (CPU), 0.17 s (Wall)] -- - Progress of 'loading chromatogram list': - - -- done [took 0.00 s (CPU), 0.00 s (Wall)] -- - --- done [took 0.18 s (CPU), 0.18 s (Wall) @ 40.66 MiB/s] -- -Progress of 'mass trace detection': - --- done [took 0.01 s (CPU), 0.01 s (Wall)] -- -Progress of 'elution peak detection': - --- done [took 0.07 s (CPU), 0.07 s (Wall)] -- -Progress of 'assembling mass traces to features': -Loading metabolite isotope model with 5% RMS error - --- done [took 0.04 s (CPU), 0.04 s (Wall)] -- --- FF-Metabo stats -- -Input traces: 1382 -Output features: 1095 (total trace count: 1382) -FeatureFinderMetabo took 0.47 s (wall), 0.90 s (CPU), 0.43 s (system), 0.47 s (user); Peak Memory Usage: 88 MB. - - -Process finished: -FeatureFinderMetabo -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML -out ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML -algorithm:common:chrom_peak_snr 4.0 -algorithm:common:noise_threshold_int 1000.0 -Total time to run command: 0.60 seconds - -Progress of 'loading mzML': - Progress of 'loading spectra list': - - 77.09 % - -- done [took 0.16 s (CPU), 0.16 s (Wall)] -- - Progress of 'loading chromatogram list': - - -- done [took 0.00 s (CPU), 0.00 s (Wall)] -- - --- done [took 0.17 s (CPU), 0.17 s (Wall) @ 43.38 MiB/s] -- -Progress of 'mass trace detection': - --- done [took 0.02 s (CPU), 0.02 s (Wall)] -- -Progress of 'elution peak detection': - --- done [took 0.07 s (CPU), 0.07 s (Wall)] -- -Progress of 'assembling mass traces to features': -Loading metabolite isotope model with 5% RMS error - --- done [took 0.05 s (CPU), 0.05 s (Wall)] -- --- FF-Metabo stats -- -Input traces: 1521 -Output features: 1203 (total trace count: 1521) -FeatureFinderMetabo took 0.51 s (wall), 0.90 s (CPU), 0.45 s (system), 0.45 s (user); Peak Memory Usage: 88 MB. - - -Total time to run 2 commands: 0.60 seconds - -Running command: -python src/python-tools/example.py ../workspaces-streamlit-template/default/topp-workflow/example.json -Waiting for command to finish... - -Process finished: -python src/python-tools/example.py ../workspaces-streamlit-template/default/topp-workflow/example.json -Total time to run command: 0.04 seconds - -Writing stdout which will get logged... -Parameters for this example Python tool: -{ - "in": [ - "../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML", - "../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML" - ], - "out": [], - "number-slider": 6, - "selectbox-example": "c", - "adavanced-input": 5, - "checkbox": true -} - - -Running command: -SiriusExport -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML -in_featureinfo ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML -out ../workspaces-streamlit-template/default/topp-workflow/results/sirius-export/sirius.ms -Waiting for command to finish... - -Process finished: -SiriusExport -in ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Control.mzML ../workspaces-streamlit-template/default/topp-workflow/input-files/mzML-files/Treatment.mzML -in_featureinfo ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Control.featureXML ../workspaces-streamlit-template/default/topp-workflow/results/feature-detection/Treatment.featureXML -out ../workspaces-streamlit-template/default/topp-workflow/results/sirius-export/sirius.ms -Total time to run command: 0.65 seconds - -Number of features to be processed: 0 -Number of additional MS2 spectra to be processed: 0 -No MS1 spectrum for this precursor. Occurred 0 times. -0 spectra were skipped due to precursor charge below -1 and above +1. -Mono charge assumed and set to charge 1 with respect to current polarity 0 times. -0 features were skipped due to feature charge below -1 and above +1. -No MS1 spectrum for this precursor. Occurred 0 times. -0 spectra were skipped due to precursor charge below -1 and above +1. -Mono charge assumed and set to charge 1 with respect to current polarity 0 times. -0 features were skipped due to feature charge below -1 and above +1. - occurred 2 times -SiriusExport took 0.61 s (wall), 1.71 s (CPU), 1.06 s (system), 0.65 s (user); Peak Memory Usage: 88 MB. - occurred 2 times - - -WORKFLOW FINISHED - """, language="neon") - - - + st.help(CommandExecutor.run_python) \ No newline at end of file diff --git a/src/Workflow.py b/src/Workflow.py index 0cc3ccd..a403eb5 100644 --- a/src/Workflow.py +++ b/src/Workflow.py @@ -27,7 +27,7 @@ def configure(self) -> None: ) with t[0]: # Parameters for FeatureFinderMetabo TOPP tool. - self.ui.input_TOPP("FeatureFinderMetabo") + self.ui.input_TOPP("FeatureFinderMetabo", custom_defaults={"algorithm:common:noise_threshold_int": 1000.0}) with t[1]: # A single checkbox widget for workflow logic. self.ui.input_widget("run-adduct-detection", False, "Adduct Detection") @@ -42,8 +42,12 @@ def configure(self) -> None: self.ui.input_python("example") def execution(self) -> None: - # Get mzML input files from self.params. - # Can be done without file manager, however, it ensures everything is correct. + # Any parameter checks, here simply checking if mzML files are selected + if not self.params["mzML-files"]: + self.logger.log("ERROR: No mzML files selected.") + return + + # Get mzML files with FileManager in_mzML = self.file_manager.get_files(self.params["mzML-files"]) # Log any messages. @@ -63,7 +67,7 @@ def execution(self) -> None: # Run MetaboliteAdductDecharger for adduct detection, with disabled logs. # Without a new file list for output, the input files will be overwritten in this case. self.executor.run_topp( - "MetaboliteAdductDecharger", {"in": out_ffm, "out_fm": out_ffm}, write_log=False + "MetaboliteAdductDecharger", {"in": out_ffm, "out_fm": out_ffm} ) # Example for a custom Python tool, which is located in src/python-tools. diff --git a/src/python-tools/example.py b/src/python-tools/example.py index 21431dc..50a7b47 100644 --- a/src/python-tools/example.py +++ b/src/python-tools/example.py @@ -35,18 +35,20 @@ }, { "key": "selectbox-example", + "name": "select something", "value": "a", "options": ["a", "b", "c"], }, { "key": "adavanced-input", + "name": "advanced parameter", "value": 5, "step_size": 5, "help": "An advanced example parameter.", "advanced": True, }, { - "key": "checkbox", "value": True + "key": "checkbox", "value": True, "name": "boolean" } ] diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 8e33cd7..6cc4930 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -26,7 +26,7 @@ def __init__(self, workflow_dir: Path, logger: Logger, parameter_manager: Parame self.parameter_manager = parameter_manager def run_multiple_commands( - self, commands: list[str], write_log: bool = True + self, commands: list[str] ) -> None: """ Executes multiple shell commands concurrently in separate threads. @@ -38,10 +38,9 @@ def run_multiple_commands( Args: commands (list[str]): A list where each element is a list representing a command and its arguments. - write_log (bool): If True, logs the execution details and outcomes of the commands. """ # Log the start of command execution - self.logger.log(f"Running {len(commands)} commands in parallel...") + self.logger.log(f"Running {len(commands)} commands in parallel...", 1) start_time = time.time() # Initialize a list to keep track of threads @@ -49,7 +48,7 @@ def run_multiple_commands( # Start a new thread for each command for cmd in commands: - thread = threading.Thread(target=self.run_command, args=(cmd, write_log)) + thread = threading.Thread(target=self.run_command, args=(cmd,)) thread.start() threads.append(thread) @@ -59,27 +58,23 @@ def run_multiple_commands( # Calculate and log the total execution time end_time = time.time() - self.logger.log( - f"Total time to run {len(commands)} commands: {end_time - start_time:.2f} seconds" - ) + self.logger.log(f"Total time to run {len(commands)} commands: {end_time - start_time:.2f} seconds", 1) - def run_command(self, command: list[str], write_log: bool = True) -> None: + def run_command(self, command: list[str]) -> None: """ Executes a specified shell command and logs its execution details. Args: command (list[str]): The shell command to execute, provided as a list of strings. - write_log (bool): If True, logs the command's output and errors. Raises: Exception: If the command execution results in any errors. """ # Ensure all command parts are strings command = [str(c) for c in command] - + # Log the execution start - self.logger.log(f"Running command:\n"+' '.join(command)+"\nWaiting for command to finish...") - + self.logger.log(f"Running command:\n"+' '.join(command)+"\nWaiting for command to finish...", 1) start_time = time.time() # Execute the command @@ -96,24 +91,22 @@ def run_command(self, command: list[str], write_log: bool = True) -> None: # Cleanup PID file pid_file_path.unlink() - + end_time = time.time() execution_time = end_time - start_time - # Format the logging prefix - self.logger.log(f"Process finished:\n"+' '.join(command)+f"\nTotal time to run command: {execution_time:.2f} seconds") + self.logger.log(f"Process finished:\n"+' '.join(command)+f"\nTotal time to run command: {execution_time:.2f} seconds", 1) # Log stdout if present - if stdout and write_log: - self.logger.log(stdout.decode()) + if stdout: + self.logger.log(stdout.decode(), 2) # Log stderr and raise an exception if errors occurred if stderr or process.returncode != 0: error_message = stderr.decode().strip() - self.logger.log(f"ERRORS OCCURRED:\n{error_message}") - raise Exception(f"Errors occurred while running command: {' '.join(command)}\n{error_message}") + self.logger.log(f"ERRORS OCCURRED:\n{error_message}", 2) - def run_topp(self, tool: str, input_output: dict, write_log: bool = True) -> None: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> None: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -130,8 +123,8 @@ def run_topp(self, tool: str, input_output: dict, write_log: bool = True) -> Non Args: tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). - write_log (bool): If True, enables logging of command execution details. - + custom_params (dict): A dictionary of custom parameters to pass to the tool. + Raises: ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. @@ -173,14 +166,31 @@ def run_topp(self, tool: str, input_output: dict, write_log: bool = True) -> Non # Add non-default TOPP tool parameters if tool in params.keys(): for k, v in params[tool].items(): - command += [f"-{k}", str(v)] + command += [f"-{k}"] + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] + # Add custom parameters + for k, v in custom_params.items(): + command += [f"-{k}"] + if v: + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] commands.append(command) + # check if a ini file has been written, if yes use it (contains custom defaults) + ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") + if ini_path.exists(): + command += ["-ini", str(ini_path)] + # Run command(s) if len(commands) == 1: - self.run_command(commands[0], write_log) + self.run_command(commands[0]) elif len(commands) > 1: - self.run_multiple_commands(commands, write_log) + self.run_multiple_commands(commands) else: raise Exception("No commands to execute.") @@ -200,7 +210,7 @@ def stop(self) -> None: shutil.rmtree(self.pid_dir, ignore_errors=True) self.logger.log("Workflow stopped.") - def run_python(self, script_file: str, input_output: dict = {}, write_log: bool = True) -> None: + def run_python(self, script_file: str, input_output: dict = {}) -> None: """ Executes a specified Python script with dynamic input and output parameters, optionally logging the execution process. The method identifies and loads @@ -217,8 +227,6 @@ def run_python(self, script_file: str, input_output: dict = {}, write_log: bool If the path is omitted, the method looks for the script in 'src/python-tools/'. The '.py' extension is appended if not present. input_output (dict, optional): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). Defaults to {}. - write_log (bool, optional): If True, the execution process is logged. This - includes any output generated by the script as well as any errors. Defaults to True. """ # Check if script file exists (can be specified without path and extension) # default location: src/python-tools/script_file @@ -240,7 +248,7 @@ def run_python(self, script_file: str, input_output: dict = {}, write_log: bool if defaults is None: self.logger.log(f"WARNING: No DEFAULTS found in {path.name}") # run command without params - self.run_command(["python", str(path)], write_log) + self.run_command(["python", str(path)]) elif isinstance(defaults, list): defaults = {entry["key"]: entry["value"] for entry in defaults} # load paramters from JSON file @@ -255,6 +263,6 @@ def run_python(self, script_file: str, input_output: dict = {}, write_log: bool with open(tmp_params_file, "w", encoding="utf-8") as f: json.dump(defaults, f, indent=4) # run command - self.run_command(["python", str(path), str(tmp_params_file)], write_log) + self.run_command(["python", str(path), str(tmp_params_file)]) # remove tmp params file tmp_params_file.unlink() \ No newline at end of file diff --git a/src/workflow/FileManager.py b/src/workflow/FileManager.py index 923ff6b..e49ef3b 100644 --- a/src/workflow/FileManager.py +++ b/src/workflow/FileManager.py @@ -175,6 +175,5 @@ def _create_results_sub_dir(self, name: str = "") -> str: while Path(self.workflow_dir, "results", name).exists(): name = self._generate_random_code(4) path = Path(self.workflow_dir, "results", name) - shutil.rmtree(path, ignore_errors=True) - path.mkdir() + path.mkdir(exist_ok=True) return str(path) diff --git a/src/workflow/Logger.py b/src/workflow/Logger.py index 8529938..4d44426 100644 --- a/src/workflow/Logger.py +++ b/src/workflow/Logger.py @@ -2,7 +2,7 @@ class Logger: """ - A simple logging class for writing messages to a log file. This class is designed + A simple logging class for writing messages to a log file. input_widgetThis class is designed to append messages to a log file in the current workflow directory, facilitating easy tracking of events, errors, or other significant occurrences in processes called during workflow execution. @@ -12,9 +12,8 @@ class Logger: """ def __init__(self, workflow_dir: Path) -> None: self.workflow_dir = workflow_dir - self.log_file = Path(self.workflow_dir, "log.txt") - def log(self, message: str) -> None: + def log(self, message: str, level: int = 0) -> None: """ Appends a given message to the log file, followed by two newline characters for readability. This method ensures that each logged message is separated @@ -22,7 +21,22 @@ def log(self, message: str) -> None: Args: message (str): The message to be logged to the file. + level (int, optional): The level of importance of the message. Defaults to 0. """ + log_dir = Path(self.workflow_dir, "logs") + if not log_dir.exists(): + log_dir.mkdir() # Write the message to the log file. - with open(self.log_file, "a", encoding="utf-8") as f: - f.write(f"{message}\n\n") + if level == 0: + with open(Path(log_dir, "minimal.log"), "a", encoding="utf-8") as f: + f.write(f"{message}\n\n") + if level <= 1: + with open(Path(log_dir, "commands-and-run-times.log"), "a", encoding="utf-8") as f: + f.write(f"{message}\n\n") + if level <= 2: + with open(Path(log_dir, "all.log"), "a", encoding="utf-8") as f: + f.write(f"{message}\n\n") + # log_types = ["minimal", "commands and run times", "tool outputs", "all"] + # for i, log_type in enumerate(log_types): + # with open(Path(log_dir, f"{log_type.replace(" ", "-")}.log"), "a", encoding="utf-8") as f: + # f.write(f"{message}\n\n") \ No newline at end of file diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index b8106f2..5e993af 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -60,13 +60,6 @@ def save_parameters(self) -> None: ini_key = key.replace(self.topp_param_prefix, "").encode() # get ini (default) value by ini_key ini_value = param.getValue(ini_key) - # need to convert bool values to string values - if isinstance(value, bool): - value = "true" if value else "false" - # convert strings with newlines to list - if isinstance(value, str): - if "\n" in value: - value = [v.encode() for v in value.split("\n")] # check if value is different from default if ini_value != value: # store non-default value diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 384ea49..e6c092d 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -10,6 +10,7 @@ import time from io import BytesIO import zipfile +from datetime import datetime class StreamlitUI: """ @@ -358,8 +359,11 @@ def format_files(input: Any) -> List[str]: def input_TOPP( self, topp_tool_name: str, - num_cols: int = 3, + num_cols: int = 4, exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + display_full_parameter_names: bool = False, + custom_defaults: dict = {}, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -369,36 +373,48 @@ def input_TOPP( Args: topp_tool_name (str): The name of the TOPP tool for which to generate inputs. num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. - exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. + exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. + include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. + display_full_parameter_names (bool, optional): Whether to display the full parameter names. Defaults to True. + custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. """ # write defaults ini files ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") if not ini_file_path.exists(): subprocess.call([topp_tool_name, "-write_ini", str(ini_file_path)]) + # update custom defaults if necessary + if custom_defaults: + param = poms.Param() + poms.ParamXMLFile().load(str(ini_file_path), param) + for key, value in custom_defaults.items(): + encoded_key = f"{topp_tool_name}:1:{key}".encode() + if encoded_key in param.keys(): + param.setValue(encoded_key, value) + poms.ParamXMLFile().store(str(ini_file_path), param) + # read into Param object param = poms.Param() poms.ParamXMLFile().load(str(ini_file_path), param) - - excluded_keys = [ - "log", - "debug", - "threads", - "no_progress", - "force", - "version", - "test", - ] + exclude_parameters - - param_dicts = [] - for key in param.keys(): - # Determine if the parameter should be included based on the conditions - if ( - b"input file" in param.getTags(key) - or b"output file" in param.getTags(key) - ) or (key.decode().split(":")[-1] in excluded_keys): - continue + if include_parameters: + valid_keys = [key for key in param.keys() if any([k.encode() in key for k in include_parameters])] + else: + excluded_keys = [ + "log", + "debug", + "threads", + "no_progress", + "force", + "version", + "test", + ] + exclude_parameters + valid_keys = [key for key in param.keys() if not (b"input file" in param.getTags(key) + or b"output file" in param.getTags(key) + or any([k.encode() in key for k in excluded_keys]))] + + params_decoded = [] + for key in valid_keys: entry = param.getEntry(key) - param_dict = { + tmp = { "name": entry.name.decode(), "key": key, "value": entry.value, @@ -406,66 +422,75 @@ def input_TOPP( "description": entry.description.decode(), "advanced": (b"advanced" in param.getTags(key)), } - param_dicts.append(param_dict) - - # Update parameter values from the JSON parameters file - json_params = self.params - if topp_tool_name in json_params: - for p in param_dicts: - name = p["key"].decode().split(":1:")[1] - if name in json_params[topp_tool_name]: - p["value"] = json_params[topp_tool_name][name] + params_decoded.append(tmp) + + # for each parameter in params_decoded + # if a parameter with custom default value exists, use that value + # else check if the parameter is already in self.params, if yes take the value from self.params + for p in params_decoded: + name = p["key"].decode().split(":1:")[1] + if topp_tool_name in self.params: + if name in self.params[topp_tool_name]: + p["value"] = self.params[topp_tool_name][name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] - # input widgets in n number of columns + # show input widgets cols = st.columns(num_cols) i = 0 - - # show input widgets - for p in param_dicts: - + + for p in params_decoded: # skip avdanced parameters if not selected if not st.session_state["advanced"] and p["advanced"]: continue key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" - + if display_full_parameter_names: + name = key.split(":1:")[1].replace("algorithm:", "").replace(":", " : ") + else: + name = p["name"] try: + # # sometimes strings with newline, handle as list + if isinstance(p["value"], str) and "\n" in p["value"]: + p["value"] = p["value"].split("\n") # bools - if p["value"] == "true" or p["value"] == "false": + if isinstance(p["value"], bool): cols[i].markdown("##") cols[i].checkbox( - p["name"], - value=(p["value"] == "true"), - help=p["description"], - key=key, - ) - - # string options - elif isinstance(p["value"], str) and p["valid_strings"]: - cols[i].selectbox( - p["name"], - options=p["valid_strings"], - index=p["valid_strings"].index(p["value"]), + name, + value=(p["value"] == "true") if type(p["value"]) == str else p["value"], help=p["description"], key=key, ) # strings elif isinstance(p["value"], str): - cols[i].text_input( - p["name"], value=p["value"], help=p["description"], key=key - ) + # string options + if p["valid_strings"]: + cols[i].selectbox( + name, + options=p["valid_strings"], + index=p["valid_strings"].index(p["value"]), + help=p["description"], + key=key, + ) + else: + cols[i].text_input( + name, value=p["value"], help=p["description"], key=key + ) # ints elif isinstance(p["value"], int): cols[i].number_input( - p["name"], value=int(p["value"]), help=p["description"], key=key + name, value=int(p["value"]), help=p["description"], key=key ) # floats elif isinstance(p["value"], float): cols[i].number_input( - p["name"], + name, value=float(p["value"]), step=1.0, help=p["description"], @@ -478,7 +503,7 @@ def input_TOPP( v.decode() if isinstance(v, bytes) else v for v in p["value"] ] cols[i].text_area( - p["name"], + name, value="\n".join(p["value"]), help=p["description"], key=key, @@ -646,16 +671,16 @@ def parameter_section(self, custom_paramter_function) -> None: ) with form: - cols = st.columns(2) + cols = st.columns(4) - cols[0].form_submit_button( + cols[2].form_submit_button( label="Save parameters", on_click=self.parameter_manager.save_parameters, type="primary", use_container_width=True, ) - if cols[1].form_submit_button( + if cols[3].form_submit_button( label="Load default parameters", use_container_width=True ): self.parameter_manager.reset_to_default_parameters() @@ -665,29 +690,58 @@ def parameter_section(self, custom_paramter_function) -> None: self.parameter_manager.save_parameters() def execution_section(self, start_workflow_function) -> None: + # Display a summary of non-default TOPP paramters and all others (custom and python scripts) + summary_text = "" + for key, value in self.params.items(): + if not isinstance(value, dict): + summary_text += f""" + +{key}: **{value}** +""" + elif value: + summary_text += f""" +**{key}**: + +""" + for k, v in value.items(): + summary_text += f""" +{key}: **{v}** + +""" + with st.expander("**Parameter Summary**"): + st.markdown(summary_text) + + c1, c2 = st.columns(2) + # Select log level, this can be changed at run time or later without re-running the workflow + log_level = c1.selectbox("log details", ["minimal", "commands and run times", "all"], key="log_level") + c2.markdown("##") if self.executor.pid_dir.exists(): - if st.button("Stop Workflow", type="primary", use_container_width=True): + if c2.button("Stop Workflow", type="primary", use_container_width=True): self.executor.stop() st.rerun() else: - st.button( + c2.button( "Start Workflow", type="primary", use_container_width=True, on_click=start_workflow_function, ) - - if self.logger.log_file.exists(): + log_path = Path(self.workflow_dir, "logs", log_level.replace(" ", "-") + ".log") + if log_path.exists(): if self.executor.pid_dir.exists(): with st.spinner("**Workflow running...**"): - with open(self.logger.log_file, "r", encoding="utf-8") as f: + with open(log_path, "r", encoding="utf-8") as f: st.code(f.read(), language="neon", line_numbers=True) time.sleep(2) st.rerun() else: - st.markdown("**Workflow log file**") - with open(self.logger.log_file, "r", encoding="utf-8") as f: - st.code(f.read(), language="neon", line_numbers=True) + st.markdown(f"**Workflow log file: {datetime.fromtimestamp(log_path.stat().st_ctime).strftime('%Y-%m-%d %H:%M')} CET**") + with open(log_path, "r", encoding="utf-8") as f: + content = f.read() + # Check if workflow finished successfully + if not "WORKFLOW FINISHED" in content: + st.error("**Errors occured, check log file.**") + st.code(content, language="neon", line_numbers=True) def results_section(self, custom_results_function) -> None: custom_results_function() diff --git a/src/workflow/WorkflowManager.py b/src/workflow/WorkflowManager.py index 3f70097..13af2fa 100644 --- a/src/workflow/WorkflowManager.py +++ b/src/workflow/WorkflowManager.py @@ -25,7 +25,7 @@ def start_workflow(self) -> None: The workflow itself needs to be a process, otherwise streamlit will wait for everything to finish before updating the UI again. """ # Delete the log file if it already exists - self.logger.log_file.unlink(missing_ok=True) + shutil.rmtree(Path(self.workflow_dir, "logs"), ignore_errors=True) # Start workflow process workflow_process = multiprocessing.Process(target=self.workflow_process) workflow_process.start()