From 7c939da4e6caf0ef560669ca922db0aa9567842b Mon Sep 17 00:00:00 2001 From: axelwalter Date: Wed, 14 Feb 2024 14:42:13 +0100 Subject: [PATCH] address review comments by JeeH-K --- .../6_\360\237\223\226_TOPP-Workflow_Docs.py" | 154 +----------------- src/Workflow.py | 6 +- src/python-tools/.gitignore | 1 + .../__pycache__/example.cpython-311.pyc | Bin 1503 -> 0 bytes src/workflow/CommandExecutor.py | 21 +-- src/workflow/ParameterManager.py | 7 +- src/workflow/StreamlitUI.py | 19 ++- src/workflow/WorkflowManager.py | 22 ++- 8 files changed, 54 insertions(+), 176 deletions(-) create mode 100644 src/python-tools/.gitignore delete mode 100644 src/python-tools/__pycache__/example.cpython-311.pyc diff --git "a/pages/6_\360\237\223\226_TOPP-Workflow_Docs.py" "b/pages/6_\360\237\223\226_TOPP-Workflow_Docs.py" index efa8756..3d6ce30 100644 --- "a/pages/6_\360\237\223\226_TOPP-Workflow_Docs.py" +++ "b/pages/6_\360\237\223\226_TOPP-Workflow_Docs.py" @@ -4,6 +4,7 @@ from src.workflow.Files import Files from src.workflow.CommandExecutor import CommandExecutor from src.common import page_setup +from inspect import getsource page_setup() @@ -70,76 +71,7 @@ with st.expander("**Complete example for custom Workflow class**", expanded=False): st.code( -""" -import streamlit as st -from .workflow.WorkflowManager import WorkflowManager -from .workflow.Files import Files - - -class TOPPWorkflow(WorkflowManager): - # Setup pages for upload, parameter settings and define workflow steps. - # For layout use any streamlit components such as tabs (as shown in example), columns, or even expanders. - def __init__(self): - # Initialize the parent class with the workflow name. - super().__init__("TOPP Workflow") - - def upload(self): - t = st.tabs(["MS data", "Example with fallback data"]) - with t[0]: - # Use the upload method from StreamlitUI to handle mzML file uploads. - self.ui.upload(key="mzML-files", name="MS data", file_type="mzML") - with t[1]: - # Example with fallback data (not used in workflow) - self.ui.upload(key="image", file_type="png", fallback="assets/OpenMS.png") - - def input(self) -> None: - # Allow users to select mzML files for the analysis. - self.ui.select_input_file("mzML-files", multiple=True) - - # Create tabs for different analysis steps. - t = st.tabs( - ["**Feature Detection**", "**Adduct Detection**", "**SIRIUS Export**"] - ) - with t[0]: - self.ui.input_TOPP("FeatureFinderMetabo") - with t[1]: - self.ui.input("run-adduct-detection", True, "Adduct Detection") - self.ui.input_TOPP("MetaboliteAdductDecharger") - with t[2]: - self.ui.input_TOPP("SiriusExport") - - def workflow(self) -> None: - # Wrap mzML files into a Files object for processing. - in_mzML = Files(self.params["mzML-files"], "mzML") - self.logger.log(f"Number of input mzML files: {len(in_mzML)}") - - # Prepare output files for feature detection. - out_ffm = Files(in_mzML, "featureXML", "feature-detection") - # Run FeatureFinderMetabo tool with input and output files. - self.executor.run_topp( - "FeatureFinderMetabo", {"in": in_mzML, "out": out_ffm}, False - ) - - # Check if adduct detection should be run. - if self.params["run-adduct-detection"]: - # Run MetaboliteAdductDecharger for adduct detection. - self.executor.run_topp( - "MetaboliteAdductDecharger", {"in": out_ffm, "out_fm": out_ffm}, False - ) - - # Combine input files for SiriusExport. - in_mzML.combine() - out_ffm.combine() - # Prepare output files for SiriusExport. - out_se = Files(["sirius-export.ms"], "ms", "sirius-export") - # Run SiriusExport tool with the combined files. - self.executor.run_topp( - "SiriusExport", - {"in": in_mzML, "in_featureinfo": out_ffm, "out": out_se}, - False, - ) - -""" +getsource(Workflow) ) @@ -161,18 +93,7 @@ def workflow(self) -> None: """) st.code( -""" -# Overwrite the upload method in your workflow class. -class YourWorkflow(WorkflowManager): - def upload(self) -> None: - t = st.tabs(["MS data", "Example with fallback data"]) - with t[0]: - # Use the upload method from StreamlitUI to handle mzML file uploads. - self.ui.upload(key="mzML-files", name="MS data", file_type="mzML") - with t[1]: - # Example with fallback data (not used in workflow). - self.ui.upload(key="image", file_type="png", fallback="assets/OpenMS.png") -""" +getsource(Workflow.upload) ) st.info("💡 Use the same **key** for parameter widgets, to select which of the uploaded files to use for analysis.") @@ -187,7 +108,7 @@ def upload(self) -> None: Generating parameter input widgets is done with the `self.ui.input` method for any parameter and the `self.ui.input_TOPP` method for TOPP tools. -**1. Choose `self.ui.input` for any paramter not-related to a TOPP tool or `self.ui.select_input_file` for any input file:** +**1. Choose `self.ui.input_widget` for any paramter not-related to a TOPP tool or `self.ui.select_input_file` for any input file:** It takes the obligatory **key** parameter. The key is used to access the parameter value in the workflow parameters dictionary `self.params`. Default values do not need to be specified in a separate file. Instead they are determined from the widgets default value automatically. Widget types can be specified or automatically determined from **default** and **options** parameters. It's suggested to add a **help** text and other parameters for numerical input. @@ -195,7 +116,7 @@ def upload(self) -> None: **2. Choose `self.ui.input_TOPP` to automatically generate complete input sections for a TOPP tool:** -It takes the obligatory **topp_tool_name** parameter and generates input widgets for each parameter present in the **ini** file (automatically created) except for input and output file parameters. For all input file parameters a widget needs to be created with `self.ui.input` with an appropriate **key**. For TOPP tool parameters only non-default values are stored. +It takes the obligatory **topp_tool_name** parameter and generates input widgets for each parameter present in the **ini** file (automatically created) except for input and output file parameters. For all input file parameters a widget needs to be created with `self.ui.select_input_file` with an appropriate **key**. For TOPP tool parameters only non-default values are stored. **3. Choose `self.ui.input_python` to automatically generate complete input sections for a custom Python tool:** @@ -221,36 +142,12 @@ def upload(self) -> None: """) st.code( -""" -def parameter(self) -> None: - # Allow users to select mzML files for the analysis. - self.ui.select_input_file("mzML-files", multiple=True) - - # Create tabs for different analysis steps. - t = st.tabs( - ["**Feature Detection**", "**Adduct Detection**", "**SIRIUS Export**", "**Python Custom Tool**"] - ) - with t[0]: - # Parameters for FeatureFinderMetabo TOPP tool. - self.ui.input_TOPP("FeatureFinderMetabo") - with t[1]: - # A single checkbox widget for workflow logic. - self.ui.input("run-adduct-detection", False, "Adduct Detection") - # Paramters for MetaboliteAdductDecharger TOPP tool. - self.ui.input_TOPP("MetaboliteAdductDecharger") - with t[2]: - # Paramters for SiriusExport TOPP tool - self.ui.input_TOPP("SiriusExport") - with t[3]: - # Generate input widgets for a custom Python tool, located at src/python-tools. - # Parameters are specified within the file in the DEFAULTS dictionary. - self.ui.input_python("example") -""" +getsource(Workflow.configure) ) st.info("💡 Access parameter widget values by their **key** in the `self.params` object, e.g. `self.params['mzML-files']` will give all selected mzML files.") with st.expander("**Code documentation**", expanded=True): - st.help(StreamlitUI.input) + st.help(StreamlitUI.input_widget) st.help(StreamlitUI.select_input_file) st.help(StreamlitUI.input_TOPP) st.help(StreamlitUI.input_python) @@ -351,42 +248,7 @@ def parameter(self) -> None: st.markdown("**Example for a complete workflow section:**") st.code( -""" -def execution(self) -> None: - # Wrap mzML files into a Files object for processing. - in_mzML = Files(self.params["mzML-files"], "mzML") - - # Log any messages. - self.logger.log(f"Number of input mzML files: {len(in_mzML)}") - - self.logger.log(in_mzML) - # Prepare output files for feature detection. - out_ffm = Files(in_mzML, "featureXML", "feature-detection") - self.logger.log(in_mzML) - - # Run FeatureFinderMetabo tool with input and output files. - self.executor.run_topp( - "FeatureFinderMetabo", input_output={"in": in_mzML, "out": out_ffm} - ) - - # Check if adduct detection should be run. - if self.params["run-adduct-detection"]: - - # Run MetaboliteAdductDecharger for adduct detection, with disabled logs. - # Without a new Files object for output, the input files will be overwritten in this case. - self.executor.run_topp( - "MetaboliteAdductDecharger", {"in": out_ffm, "out_fm": out_ffm}, write_log=False - ) - - # Example for a custom Python tool, which is located in src/python-tools. - self.executor.run_python("example", {"in": in_mzML}) - - # Prepare output file for SiriusExport. - out_se = Files(["sirius-export.ms"], "ms", "sirius-export") - - # Run SiriusExport tool with the collected files. - self.executor.run_topp("SiriusExport", {"in": in_mzML.collect(), "in_featureinfo": out_ffm.collect(), "out": out_se}) - """ +getsource(Workflow.execution) ) with st.expander("**Example output (truncated) of the workflow code above**"): diff --git a/src/Workflow.py b/src/Workflow.py index 093e923..b6c035c 100644 --- a/src/Workflow.py +++ b/src/Workflow.py @@ -5,11 +5,11 @@ class Workflow(WorkflowManager): # Setup pages for upload, parameter, execution and results. # For layout use any streamlit components such as tabs (as shown in example), columns, or even expanders. - def __init__(self): + def __init__(self) -> None: # Initialize the parent class with the workflow name. super().__init__("TOPP Workflow") - def upload(self): + def upload(self)-> None: t = st.tabs(["MS data", "Example with fallback data"]) with t[0]: # Use the upload method from StreamlitUI to handle mzML file uploads. @@ -31,7 +31,7 @@ def configure(self) -> None: self.ui.input_TOPP("FeatureFinderMetabo") with t[1]: # A single checkbox widget for workflow logic. - self.ui.input("run-adduct-detection", False, "Adduct Detection") + self.ui.input_widget("run-adduct-detection", False, "Adduct Detection") # Paramters for MetaboliteAdductDecharger TOPP tool. self.ui.input_TOPP("MetaboliteAdductDecharger") with t[2]: diff --git a/src/python-tools/.gitignore b/src/python-tools/.gitignore new file mode 100644 index 0000000..ed8ebf5 --- /dev/null +++ b/src/python-tools/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file diff --git a/src/python-tools/__pycache__/example.cpython-311.pyc b/src/python-tools/__pycache__/example.cpython-311.pyc deleted file mode 100644 index a3135d29240c08f98c4375ee785b72e9b0418e47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1503 zcmaJ=%}*Og6rb^q&DzGbmlzT#HJc_1aTMF7z@lyrs}$G5LaEd9Q9F`HZ4k0;#->0CJZ%+VRO_mVUpFR-rP5B z`hNso#%#CNh*NS6?&Ek9gQIG#tLsFgZHKu11ka`~TH6=(c!dX6M{Q|l_nn1GEwyf$ zu3#v$#DJ5h&Ygw|-+CM@YvV%J>VBc9?CgVgDo zwOu0HTFW*FgIZB?CWI4QW3bK-f_uO^X4zEd(Y>sOwJpu86I{|o*ZLx;S6DGs4G;9{ zkWaNWhi653y2bbzm7@sKC^vcJ!sL3B)HiuD#n_o8lWVn>rkk}|UwlyvI^O&ZWt;rQZ=kbgWz(x?A?4c3O;yI+SJ{5i4C8ZT9)xiU({HHETNm2 zn6#K?Xv4L52i>mA`K{$bA%Co(m?YLM0hbgh0R)$XyGRd~oT*y}NsN z!!aZ-1dM~+RpB`~CEW5?!gNAh3CMxm%;CLIiV1~)#o6)LjvU?rXuRkBE=~K=bdb#S z^1b?QbMO1z&HcsWHUQtW?#28A-am-FJbhzfHU1i zyVKLti`1bE;;EhceroX;EuEkxA1wtaDHKb`X!!&!`)D~*^ltU8@7~&bdbr}JZXBm> loNz?*CurUu%`gkl46nr%|COVhvQm}Uyc{1@PTp^pFn diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 614faf2..767300a 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -124,10 +124,7 @@ def run_topp(self, tool: str, input_output: dict, write_log: bool = True) -> Non Args: tool (str): The executable name or path of the tool. - input_output (dict): A dictionary specifying the input and output - parameters and their corresponding files. The files - can be specified as single paths (strings) or lists - of paths for batch processing. + input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). write_log (bool): If True, enables logging of command execution details. Raises: @@ -149,20 +146,23 @@ def run_topp(self, tool: str, input_output: dict, write_log: bool = True) -> Non commands = [] # Load parameters for non-defaults - params = self.parameter_manager.load_parameters() + params = self.parameter_manager.get_parameters_from_json() # Construct commands for each process for i in range(n_processes): command = [tool] # Add input/output files for k in input_output.keys(): + # add key as parameter name command += [f"-{k}"] + # get value from input_output dictionary value = input_output[k] - if isinstance(value, Files): - value = value.files + # when multiple input/output files exist (e.g., multiple mzMLs and featureXMLs), but only one additional input file (e.g., one input database file) if len(value) == 1: i = 0 + # when the entry is a list of collected files to be passed as one [["sample1", "sample2"]] if isinstance(value[i], list): command += value[i] + # standard case, files was a list of strings, take the file name at index else: command += [value[i]] # Add non-default TOPP tool parameters @@ -211,10 +211,7 @@ def run_python(self, script_file: str, input_output: dict = {}, write_log: bool script_file (str): The name or path of the Python script to be executed. If the path is omitted, the method looks for the script in 'src/python-tools/'. The '.py' extension is appended if not present. - input_output (dict, optional): A dictionary mapping parameter names to their - values. These parameters are passed to the script, overriding any default - values specified within the script. If a parameter value is an instance of - a Files object, its 'files' attribute is used as the parameter value. Defaults to {}. + input_output (dict, optional): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). Defaults to {}. write_log (bool, optional): If True, the execution process is logged. This includes any output generated by the script as well as any errors. Defaults to True. """ @@ -236,7 +233,7 @@ def run_python(self, script_file: str, input_output: dict = {}, write_log: bool spec.loader.exec_module(module) defaults = {entry["key"]: entry["value"] for entry in getattr(module, "DEFAULTS", None)} # load paramters from JSON file - params = {k: v for k, v in self.parameter_manager.load_parameters().items() if path.name in k} + params = {k: v for k, v in self.parameter_manager.get_parameters_from_json().items() if path.name in k} # update defaults for k, v in params.items(): defaults[k.replace(f"{path.name}:", "")] = v diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index 19efb73..b8106f2 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -62,10 +62,7 @@ def save_parameters(self) -> None: ini_value = param.getValue(ini_key) # need to convert bool values to string values if isinstance(value, bool): - if value == True: - value = "true" - elif value == False: - value = "false" + value = "true" if value else "false" # convert strings with newlines to list if isinstance(value, str): if "\n" in value: @@ -78,7 +75,7 @@ def save_parameters(self) -> None: with open(self.params_file, "w", encoding="utf-8") as f: json.dump(json_params, f, indent=4) - def load_parameters(self) -> None: + def get_parameters_from_json(self) -> None: """ Loads parameters from the JSON file if it exists and returns them as a dictionary. If the file does not exist, it returns an empty dictionary. diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 5446dec..0f10a99 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -29,7 +29,7 @@ class StreamlitUI: def __init__(self, workflow_manager): self.workflow_manager = workflow_manager self.workflow_dir = workflow_manager.workflow_dir - self.params = self.workflow_manager.parameter_manager.load_parameters() + self.params = self.workflow_manager.parameter_manager.get_parameters_from_json() def upload( self, @@ -104,6 +104,7 @@ def upload( for i, f in enumerate(files): my_bar.progress((i + 1) / len(files)) shutil.copy(f, Path(files_dir, f.name)) + my_bar.empty() st.success("Successfully copied files!") if fallback: @@ -172,7 +173,7 @@ def select_input_file( self.params[key] = [f for f in self.params[key] if f in options] widget_type = "multiselect" if multiple else "selectbox" - self.input( + self.input_widget( key, name=name, widget_type=widget_type, @@ -180,7 +181,7 @@ def select_input_file( display_file_path=display_file_path, ) - def input( + def input_widget( self, key: str, default: Any = None, @@ -328,7 +329,7 @@ def format_files(input: Any) -> List[str]: if isinstance(value, bool): st.checkbox(name, value=value, key=key, help=help) elif isinstance(value, (int, float)): - self.input( + self.input_widget( key, value, widget_type="number", @@ -339,7 +340,7 @@ def format_files(input: Any) -> List[str]: help=help, ) elif (isinstance(value, str) or value == None) and options is not None: - self.input( + self.input_widget( key, value, widget_type="selectbox", @@ -348,7 +349,7 @@ def format_files(input: Any) -> List[str]: help=help, ) elif isinstance(value, list) and options is not None: - self.input( + self.input_widget( key, value, widget_type="multiselect", @@ -357,9 +358,9 @@ def format_files(input: Any) -> List[str]: help=help, ) elif isinstance(value, bool): - self.input(key, value, widget_type="checkbox", name=name, help=help) + self.input_widget(key, value, widget_type="checkbox", name=name, help=help) else: - self.input(key, value, widget_type="text", name=name, help=help) + self.input_widget(key, value, widget_type="text", name=name, help=help) else: st.error(f"Unsupported widget type '{widget_type}'") @@ -575,7 +576,7 @@ def input_python( with cols[i]: if isinstance(value, bool): st.markdown("#") - self.input( + self.input_widget( key=key, default=value, name=name, diff --git a/src/workflow/WorkflowManager.py b/src/workflow/WorkflowManager.py index 696b7dd..51cc9d1 100644 --- a/src/workflow/WorkflowManager.py +++ b/src/workflow/WorkflowManager.py @@ -17,9 +17,14 @@ def __init__(self, name: str = "Workflow Base"): self.logger = Logger(self.workflow_dir) self.executor = CommandExecutor(self.workflow_dir, self.logger, self.parameter_manager) self.ui = StreamlitUI(self) - self.params = self.parameter_manager.load_parameters() + self.params = self.parameter_manager.get_parameters_from_json() + def start_workflow(self) -> None: + """ + Starts the workflow process and adds its process id to the pid directory. + The workflow itself needs to be a process, otherwise streamlit will wait for everything to finish before updating the UI again. + """ # Delete the log file if it already exists self.logger.log_file.unlink(missing_ok=True) # Start workflow process @@ -30,6 +35,9 @@ def start_workflow(self) -> None: Path(self.executor.pid_dir, str(workflow_process.pid)).touch() def workflow_process(self) -> None: + """ + Workflow process. Logs start and end of the workflow and calls the execution method where all steps are defined. + """ try: self.logger.log("Starting workflow...") results_dir = Path(self.workflow_dir, "results") @@ -45,24 +53,36 @@ def workflow_process(self) -> None: def upload(self) -> None: + """ + Add your file upload widgets here + """ ################################### # Add your file upload widgets here ################################### pass def configure(self) -> None: + """ + Add your input widgets here + """ ################################### # Add your input widgets here ################################### pass def execution(self) -> None: + """ + Add your workflow steps here + """ ################################### # Add your workflow steps here ################################### pass def results(self) -> None: + """ + Display results here + """ ################################### # Display results here ###################################