From 2e6c43b82749a83f960e7b355ab01dfee0a4a57d Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:27:44 -0400 Subject: [PATCH 1/2] allow for using pipestat.summarize, align with pipestat 0.4.0 --- looper/looper.py | 68 ++++++++++++++++++++++--------- looper/project.py | 23 ++++++----- requirements/requirements-all.txt | 2 +- 3 files changed, 61 insertions(+), 32 deletions(-) diff --git a/looper/looper.py b/looper/looper.py index ebbb82bda..48368d90d 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -577,28 +577,56 @@ def __call__(self, args): # initialize the report builder p = self.prj project_level = args.project + + # if args.project: + # psms = self.prj.get_pipestat_managers(project_level=True) + # for pipeline_name, psm in psms.items(): + # s = psm.get_status() or "unknown" + # status.setdefault(pipeline_name, {}) + # status[pipeline_name][self.prj.name] = s + # _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}") + # else: + # for sample in self.prj.samples: + # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name) + # for pipeline_name, psm in psms.items(): + # s = psm.get_status() + # status.setdefault(pipeline_name, {}) + # status[pipeline_name][sample.sample_name] = s + # _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}") + if project_level: - html_report_builder_project = HTMLReportBuilderProject(prj=p) - self.counter = LooperCounter(len(p.project_pipeline_interfaces)) - for piface in p.project_pipeline_interface_sources: - pn = PipelineInterface(piface).pipeline_name - _LOGGER.info( - self.counter.show(name=p.name, type="project", pipeline_name=pn) - ) - # Do the stats and object summarization. - # run the report builder. a set of HTML pages is produced - report_path = html_report_builder_project(piface_source=piface) - _LOGGER.info( - f"Project-level pipeline '{pn}' HTML report: {report_path}" - ) + psms = self.prj.get_pipestat_managers(project_level=True) + print(psms) + for name, psm in psms.items(): + #Summarize will generate the static HTML Report Function + psm.summarize() + # html_report_builder_project = HTMLReportBuilderProject(prj=p) + # self.counter = LooperCounter(len(p.project_pipeline_interfaces)) + # for piface in p.project_pipeline_interface_sources: + # pn = PipelineInterface(piface).pipeline_name + # _LOGGER.info( + # self.counter.show(name=p.name, type="project", pipeline_name=pn) + # ) + # # Do the stats and object summarization. + # # run the report builder. a set of HTML pages is produced + # report_path = html_report_builder_project(piface_source=piface) + # _LOGGER.info( + # f"Project-level pipeline '{pn}' HTML report: {report_path}" + # ) else: - html_report_builder = HTMLReportBuilder(prj=self.prj) - for sample_piface_source in self.prj.pipeline_interface_sources: - # Do the stats and object summarization. - pn = PipelineInterface(sample_piface_source).pipeline_name - # run the report builder. a set of HTML pages is produced - report_path = html_report_builder(pipeline_name=pn) - _LOGGER.info(f"Sample-level pipeline '{pn}' HTML report: {report_path}") + for sample in p.prj.samples: + psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name) + print(psms) + for name, psm in psms.items(): + # Summarize will generate the static HTML Report Function + psm.summarize() + # html_report_builder = HTMLReportBuilder(prj=self.prj) + # for sample_piface_source in self.prj.pipeline_interface_sources: + # # Do the stats and object summarization. + # pn = PipelineInterface(sample_piface_source).pipeline_name + # # run the report builder. a set of HTML pages is produced + # report_path = html_report_builder(pipeline_name=pn) + # _LOGGER.info(f"Sample-level pipeline '{pn}' HTML report: {report_path}") class Tabulator(Executor): diff --git a/looper/project.py b/looper/project.py index 1085f1b94..54ed60b70 100644 --- a/looper/project.py +++ b/looper/project.py @@ -490,7 +490,8 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): :return str: retrieved configuration value """ if pipestat_sect is not None and attr_name in pipestat_sect: - return getattr(object, pipestat_sect[attr_name]) + return pipestat_sect[attr_name] + #return getattr(object, pipestat_sect[attr_name]) try: return getattr(object, default) except AttributeError: @@ -527,13 +528,13 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): ) pipestat_config = self._resolve_path_with_cfg(pth=pipestat_config) - namespace = _get_val_from_attr( - pipestat_section, - self.config if project_level else self.get_sample(sample_name), - PIPESTAT_NAMESPACE_ATTR_KEY, - "name" if project_level else self.sample_table_index, - pipestat_config and os.path.exists(pipestat_config), - ) + # project_name = _get_val_from_attr( + # pipestat_section, + # self.config if project_level else self.get_sample(sample_name), + # PIPESTAT_NAMESPACE_ATTR_KEY, + # "name" if project_level else self.sample_table_index, + # pipestat_config and os.path.exists(pipestat_config), + # ) results_file_path = _get_val_from_attr( pipestat_section, self.config if project_level else self.get_sample(sample_name), @@ -557,10 +558,10 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): else f"{piface.pipeline_name}_{'_'.join(self.amendments)}" ) ret[piface.pipeline_name] = { - "namespace": namespace, - "config": pipestat_config, + # "project_name": project_name, + "config_file": pipestat_config, "results_file_path": results_file_path, - "record_identifier": rec_id, + "sample_name": rec_id, "schema_path": piface.get_pipeline_schemas(OUTPUT_SCHEMA_KEY), } return ret diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 19afd0728..64d948682 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -6,7 +6,7 @@ logmuse>=0.2.0 pandas>=2.0.2 pephubclient peppy>=0.35.4 -pipestat>=0.1.0 +pipestat>=0.4.0 pyyaml>=3.12 rich>=9.10.0 ubiquerg>=0.5.2 From 0f7f5b9351320ea1a28796bbb6fffef571af76d7 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:41:34 -0400 Subject: [PATCH 2/2] clean up code, update usage doc --- docs/usage.md | 300 ++++++++++++++++++++++++++++------------------ looper/looper.py | 38 +----- looper/project.py | 10 +- 3 files changed, 187 insertions(+), 161 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index cd9a5c588..56aa6d688 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -26,16 +26,16 @@ Each task is controlled by one of the following commands: `run`, `rerun`, `runp` Here you can see the command-line usage instructions for the main looper command and for each subcommand: ## `looper --help` ```console -version: 1.2.0-dev -usage: looper [-h] [--version] [--logfile LOGFILE] [--verbosity {0,1,2,3,4}] - [--dbg] - {run,rerun,runp,table,report,destroy,check,clean,inspect,init} +version: 1.4.0-dev +usage: looper [-h] [--version] [--logfile LOGFILE] [--dbg] [--silent] + [--verbosity V] [--logdev] + {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface} ... looper - A project job submission engine and project manager. positional arguments: - {run,rerun,runp,table,report,destroy,check,clean,inspect,init} + {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface} run Run or submit sample jobs. rerun Resubmit sample jobs with failed flags. runp Run or submit project jobs. @@ -46,14 +46,16 @@ positional arguments: clean Run clean scripts of already processed jobs. inspect Print information about a project. init Initialize looper dotfile. + init-piface Initialize generic pipeline interface. -optional arguments: +options: -h, --help show this help message and exit --version show program's version number and exit --logfile LOGFILE Optional output file for looper logs (default: None) - --verbosity {0,1,2,3,4} - Choose level of verbosity (default: None) --dbg Turn on debug mode (default: False) + --silent Silence logging. Overrides verbosity. + --verbosity V Set logging level (1-5 or logging module level name) + --logdev Expand content of logging message format. For subcommand-specific options, type: 'looper -h' https://github.com/pepkit/looper @@ -61,35 +63,39 @@ https://github.com/pepkit/looper ## `looper run --help` ```console -usage: looper run [-h] [-i] [-d] [-t S] [-l N] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] - [-s S] [-c K [K ...]] [-u X] [-n N] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper run [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S] + [-c K [K ...]] [-u X] [-n N] [-S YAML [YAML ...]] [-P YAML [YAML ...]] + [-l N] [-k N] [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl + [I ...]] [-a A [A ...]] [config_file] Run or submit sample jobs. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: +options: -h, --help show this help message and exit -i, --ignore-flags Ignore run status flags? Default=False -d, --dry-run Don't actually submit the jobs. Default=False -t S, --time-delay S Time delay in seconds between job submissions - -l N, --limit N Limit to n samples -x S, --command-extra S String to append to every command -y S, --command-extra-override S Same as command-extra, but overrides values in PEP -f, --skip-file-checks Do not perform input file checks -u X, --lump X Total input file size (GB) to batch into one job -n N, --lumpn N Number of commands to batch into one job + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file -a A [A ...], --amend A [A ...] List of amendments to activate divvy arguments: Configure divvy to change computing settings --divvy DIVCFG Path to divvy configuration file. Default=$DIVCFG env - variable. Currently: /Users/mstolarczyk/Uczelnia/UVA/ - code//divcfg/uva_rivanna.yaml + variable. Currently: not set -p P, --package P Name of computing resource package to use -s S, --settings S Path to a YAML settings file with compute settings -c K [K ...], --compute K [K ...] List of key-value pairs (k1=v1) @@ -97,41 +103,46 @@ divvy arguments: sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper runp --help` ```console -usage: looper runp [-h] [-i] [-d] [-t S] [-l N] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] - [-s S] [-c K [K ...]] [-g K] [--sel-attr ATTR] [--sel-excl [E [E ...]] - | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper runp [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S] + [-c K [K ...]] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N] + [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]] + [-a A [A ...]] [config_file] Run or submit project jobs. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: +options: -h, --help show this help message and exit -i, --ignore-flags Ignore run status flags? Default=False -d, --dry-run Don't actually submit the jobs. Default=False -t S, --time-delay S Time delay in seconds between job submissions - -l N, --limit N Limit to n samples -x S, --command-extra S String to append to every command -y S, --command-extra-override S Same as command-extra, but overrides values in PEP -f, --skip-file-checks Do not perform input file checks + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file -a A [A ...], --amend A [A ...] List of amendments to activate divvy arguments: Configure divvy to change computing settings --divvy DIVCFG Path to divvy configuration file. Default=$DIVCFG env - variable. Currently: /Users/mstolarczyk/Uczelnia/UVA/ - code//divcfg/uva_rivanna.yaml + variable. Currently: not set -p P, --package P Name of computing resource package to use -s S, --settings S Path to a YAML settings file with compute settings -c K [K ...], --compute K [K ...] List of key-value pairs (k1=v1) @@ -139,43 +150,48 @@ divvy arguments: sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper rerun --help` ```console -usage: looper rerun [-h] [-i] [-d] [-t S] [-l N] [-x S] [-y S] [-f] [--divvy DIVCFG] - [-p P] [-s S] [-c K [K ...]] [-u X] [-n N] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper rerun [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] + [-s S] [-c K [K ...]] [-u X] [-n N] [-S YAML [YAML ...]] + [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR] + [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]] [config_file] Resubmit sample jobs with failed flags. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: +options: -h, --help show this help message and exit -i, --ignore-flags Ignore run status flags? Default=False -d, --dry-run Don't actually submit the jobs. Default=False -t S, --time-delay S Time delay in seconds between job submissions - -l N, --limit N Limit to n samples -x S, --command-extra S String to append to every command -y S, --command-extra-override S Same as command-extra, but overrides values in PEP -f, --skip-file-checks Do not perform input file checks -u X, --lump X Total input file size (GB) to batch into one job -n N, --lumpn N Number of commands to batch into one job + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file -a A [A ...], --amend A [A ...] List of amendments to activate divvy arguments: Configure divvy to change computing settings --divvy DIVCFG Path to divvy configuration file. Default=$DIVCFG env - variable. Currently: /Users/mstolarczyk/Uczelnia/UVA/ - code//divcfg/uva_rivanna.yaml + variable. Currently: not set -p P, --package P Name of computing resource package to use -s S, --settings S Path to a YAML settings file with compute settings -c K [K ...], --compute K [K ...] List of key-value pairs (k1=v1) @@ -183,178 +199,232 @@ divvy arguments: sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper report --help` ```console -usage: looper report [-h] [-g K] [--sel-attr ATTR] [--sel-excl [E [E ...]] | --sel-incl - [I [I ...]]] [-a A [A ...]] +usage: looper report [-h] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N] + [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]] + [-a A [A ...]] [--project] [config_file] Create browsable HTML report of project results. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: - -h, --help show this help message and exit - -a A [A ...], --amend A [A ...] List of amendments to activate +options: + -h, --help show this help message and exit + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -a A [A ...], --amend A [A ...] List of amendments to activate + --project Process project-level pipelines sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle - --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index + --sel-attr ATTR Attribute for sample exclusion OR inclusion + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper table --help` ```console -usage: looper table [-h] [-g K] [--sel-attr ATTR] [--sel-excl [E [E ...]] | --sel-incl - [I [I ...]]] [-a A [A ...]] +usage: looper table [-h] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N] + [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]] + [-a A [A ...]] [--project] [config_file] Write summary stats table for project samples. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: - -h, --help show this help message and exit - -a A [A ...], --amend A [A ...] List of amendments to activate +options: + -h, --help show this help message and exit + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -a A [A ...], --amend A [A ...] List of amendments to activate + --project Process project-level pipelines sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle - --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index + --sel-attr ATTR Attribute for sample exclusion OR inclusion + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper inspect --help` ```console -usage: looper inspect [-h] [-n S [S ...]] [-l L] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper inspect [-h] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N] + [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]] + [-a A [A ...]] [--sample-names [SAMPLE_NAMES ...]] + [--attr-limit ATTR_LIMIT] [config_file] Print information about a project. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: - -h, --help show this help message and exit - -n S [S ...], --snames S [S ...] Name of the samples to inspect - -l L, --attr-limit L Number of sample attributes to display - -a A [A ...], --amend A [A ...] List of amendments to activate +options: + -h, --help show this help message and exit + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -a A [A ...], --amend A [A ...] List of amendments to activate + --sample-names [SAMPLE_NAMES ...] Names of the samples to inspect + --attr-limit ATTR_LIMIT Number of attributes to display sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle - --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index + --sel-attr ATTR Attribute for sample exclusion OR inclusion + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper init --help` ```console -usage: looper init [-h] [-f] config_file +usage: looper init [-h] [-f] [-o DIR] [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-p] + config_file Initialize looper dotfile. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) -optional arguments: - -h, --help show this help message and exit - -f, --force Force overwrite +options: + -h, --help show this help message and exit + -f, --force Force overwrite + -o DIR, --output-dir DIR + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -p, --piface Generates generic pipeline interface ``` ## `looper destroy --help` ```console -usage: looper destroy [-h] [-d] [--force-yes] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper destroy [-h] [-d] [--force-yes] [-S YAML [YAML ...]] [-P YAML [YAML ...]] + [-l N] [-k N] [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl + [I ...]] [-a A [A ...]] [config_file] Remove output files of the project. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: - -h, --help show this help message and exit - -d, --dry-run Don't actually submit the jobs. Default=False - --force-yes Provide upfront confirmation of destruction intent, to - skip console query. Default=False - -a A [A ...], --amend A [A ...] List of amendments to activate +options: + -h, --help show this help message and exit + -d, --dry-run Don't actually submit the jobs. Default=False + --force-yes Provide upfront confirmation of destruction intent, + to skip console query. Default=False + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -a A [A ...], --amend A [A ...] List of amendments to activate sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle - --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index + --sel-attr ATTR Attribute for sample exclusion OR inclusion + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper check --help` ```console -usage: looper check [-h] [-A] [-f [F [F ...]]] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper check [-h] [--describe-codes] [--itemized] [-f [F ...]] [-S YAML [YAML ...]] + [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR] + [--sel-excl [E ...] | --sel-incl [I ...]] [-a A [A ...]] [--project] [config_file] Check flag status of current runs. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: +options: -h, --help show this help message and exit - -A, --all-folders Check status for all output folders, not just for - samples specified in the config. Default=False - -f [F [F ...]], --flags [F [F ...]] - Check on only these flags/status values + --describe-codes Show status codes description + --itemized Show a detailed, by sample statuses + -f [F ...], --flags [F ...] Check on only these flags/status values + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file -a A [A ...], --amend A [A ...] List of amendments to activate + --project Process project-level pipelines sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` ## `looper clean --help` ```console -usage: looper clean [-h] [-d] [--force-yes] [-g K] [--sel-attr ATTR] - [--sel-excl [E [E ...]] | --sel-incl [I [I ...]]] [-a A [A ...]] +usage: looper clean [-h] [-d] [--force-yes] [-S YAML [YAML ...]] [-P YAML [YAML ...]] + [-l N] [-k N] [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl + [I ...]] [-a A [A ...]] [config_file] Run clean scripts of already processed jobs. positional arguments: - config_file Project configuration file (YAML) + config_file Project configuration file (YAML) or pephub registry + path. -optional arguments: - -h, --help show this help message and exit - -d, --dry-run Don't actually submit the jobs. Default=False - --force-yes Provide upfront confirmation of destruction intent, to - skip console query. Default=False - -a A [A ...], --amend A [A ...] List of amendments to activate +options: + -h, --help show this help message and exit + -d, --dry-run Don't actually submit the jobs. Default=False + --force-yes Provide upfront confirmation of destruction intent, + to skip console query. Default=False + -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...] + Path to looper sample config file + -P YAML [YAML ...], --project-pipeline-interfaces YAML [YAML ...] + Path to looper project config file + -a A [A ...], --amend A [A ...] List of amendments to activate sample selection arguments: Specify samples to include or exclude based on sample attribute values - -g K, --toggle-key K Sample attribute specifying toggle. Default: toggle - --sel-attr ATTR Attribute for sample exclusion OR inclusion - --sel-excl [E [E ...]] Exclude samples with these values - --sel-incl [I [I ...]] Include only samples with these values + -l N, --limit N Limit to n samples + -k N, --skip N Skip samples by numerical index + --sel-attr ATTR Attribute for sample exclusion OR inclusion + --sel-excl [E ...] Exclude samples with these values + --sel-incl [I ...] Include only samples with these values ``` + diff --git a/looper/looper.py b/looper/looper.py index 48368d90d..fa3c62696 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -578,41 +578,12 @@ def __call__(self, args): p = self.prj project_level = args.project - # if args.project: - # psms = self.prj.get_pipestat_managers(project_level=True) - # for pipeline_name, psm in psms.items(): - # s = psm.get_status() or "unknown" - # status.setdefault(pipeline_name, {}) - # status[pipeline_name][self.prj.name] = s - # _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}") - # else: - # for sample in self.prj.samples: - # psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name) - # for pipeline_name, psm in psms.items(): - # s = psm.get_status() - # status.setdefault(pipeline_name, {}) - # status[pipeline_name][sample.sample_name] = s - # _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}") - if project_level: psms = self.prj.get_pipestat_managers(project_level=True) print(psms) for name, psm in psms.items(): - #Summarize will generate the static HTML Report Function + # Summarize will generate the static HTML Report Function psm.summarize() - # html_report_builder_project = HTMLReportBuilderProject(prj=p) - # self.counter = LooperCounter(len(p.project_pipeline_interfaces)) - # for piface in p.project_pipeline_interface_sources: - # pn = PipelineInterface(piface).pipeline_name - # _LOGGER.info( - # self.counter.show(name=p.name, type="project", pipeline_name=pn) - # ) - # # Do the stats and object summarization. - # # run the report builder. a set of HTML pages is produced - # report_path = html_report_builder_project(piface_source=piface) - # _LOGGER.info( - # f"Project-level pipeline '{pn}' HTML report: {report_path}" - # ) else: for sample in p.prj.samples: psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name) @@ -620,13 +591,6 @@ def __call__(self, args): for name, psm in psms.items(): # Summarize will generate the static HTML Report Function psm.summarize() - # html_report_builder = HTMLReportBuilder(prj=self.prj) - # for sample_piface_source in self.prj.pipeline_interface_sources: - # # Do the stats and object summarization. - # pn = PipelineInterface(sample_piface_source).pipeline_name - # # run the report builder. a set of HTML pages is produced - # report_path = html_report_builder(pipeline_name=pn) - # _LOGGER.info(f"Sample-level pipeline '{pn}' HTML report: {report_path}") class Tabulator(Executor): diff --git a/looper/project.py b/looper/project.py index 54ed60b70..84d2006a2 100644 --- a/looper/project.py +++ b/looper/project.py @@ -491,7 +491,6 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): """ if pipestat_sect is not None and attr_name in pipestat_sect: return pipestat_sect[attr_name] - #return getattr(object, pipestat_sect[attr_name]) try: return getattr(object, default) except AttributeError: @@ -528,13 +527,7 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): ) pipestat_config = self._resolve_path_with_cfg(pth=pipestat_config) - # project_name = _get_val_from_attr( - # pipestat_section, - # self.config if project_level else self.get_sample(sample_name), - # PIPESTAT_NAMESPACE_ATTR_KEY, - # "name" if project_level else self.sample_table_index, - # pipestat_config and os.path.exists(pipestat_config), - # ) + results_file_path = _get_val_from_attr( pipestat_section, self.config if project_level else self.get_sample(sample_name), @@ -558,7 +551,6 @@ def _get_val_from_attr(pipestat_sect, object, attr_name, default, no_err=False): else f"{piface.pipeline_name}_{'_'.join(self.amendments)}" ) ret[piface.pipeline_name] = { - # "project_name": project_name, "config_file": pipestat_config, "results_file_path": results_file_path, "sample_name": rec_id,