Merge pull request #447 from pepkit/dev

v1.7.0 release

.github/workflows/python-publish.yml

@@ -10,7 +10,10 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-
+    name: upload release to PyPI
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -27,4 +30,5 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           python setup.py sdist bdist_wheel
-          twine upload dist/*
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

docs/changelog.md

@@ -2,6 +2,16 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+## [1.7.0] -- 2024-01-26
+
+### Added
+- `--portable` flag to `looper report` to create a portable version of the html report
+- `--lump-j` allows grouping samples into a defined number of jobs
+
+### Changed
+- `--lumpn` is now `--lump-n`
+- `--lump` is now `--lump-s`
+
 ## [1.6.0] -- 2023-12-22
 
 ### Added

docs/looper-report.md

@@ -6,6 +6,8 @@ Looper can create a browsable html report of all project results using the comma
 looper report --looper-config .your_looper_config.yaml
 ```
 
+Beginning in Looper 1.7.0, the ``--portable`` flag can be used to create a shareable, zipped version of the html report.
+
 An example html report out put can be found here: [PEPATAC Gold Summary](https://pepatac.databio.org/en/latest/files/examples/gold/gold_summary.html)
 
 Note: pipestat must be configured by looper to perform this operation. Please see the pipestat section for more information: [Using pipestat](pipestat.md)

docs/usage.md

@@ -26,7 +26,7 @@ Each task is controlled by one of the following commands: `run`, `rerun`, `runp`
 Here you can see the command-line usage instructions for the main looper command and for each subcommand:
 ## `looper --help`
 ```console
-version: 1.6.0
+version: 1.7.0
 usage: looper [-h] [--version] [--logfile LOGFILE] [--dbg] [--silent]
               [--verbosity V] [--logdev] [--commands]
               {run,rerun,runp,table,report,destroy,check,clean,inspect,init,init-piface,link}
@@ -57,7 +57,7 @@ options:
   --silent              Silence logging. Overrides verbosity.
   --verbosity V         Set logging level (1-5 or logging module level name)
   --logdev              Expand content of logging message format.
-  --commands            show program's primary commands
+  --commands            show program's version number and exit
 
 For subcommand-specific options, type: 'looper <subcommand> -h'
 https://github.com/pepkit/looper
@@ -66,7 +66,7 @@ https://github.com/pepkit/looper
 ## `looper run --help`
 ```console
 usage: looper run [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P] [-s S]
-                  [-c K [K ...]] [-u X] [-n N] [--looper-config LOOPER_CONFIG]
+                  [-c K [K ...]] [-u X] [-n N] [-j J] [--looper-config LOOPER_CONFIG]
                   [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
                   [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
                   [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
@@ -86,8 +86,11 @@ options:
   -x S, --command-extra S            String to append to every command
   -y S, --command-extra-override S   Same as command-extra, but overrides values in PEP
   -f, --skip-file-checks             Do not perform input file checks
-  -u X, --lump X                     Total input file size (GB) to batch into one job
-  -n N, --lumpn N                    Number of commands to batch into one job
+  -u X, --lump-s X                   Lump by size: total input file size (GB) to batch
+                                     into one job
+  -n N, --lump-n N                   Lump by number: number of samples to batch into one
+                                     job
+  -j J, --lump-j J                   Lump samples into number of jobs.
   --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
   -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
                                      Path to looper sample config file
@@ -170,10 +173,11 @@ sample selection arguments:
 ## `looper rerun --help`
 ```console
 usage: looper rerun [-h] [-i] [-d] [-t S] [-x S] [-y S] [-f] [--divvy DIVCFG] [-p P]
-                    [-s S] [-c K [K ...]] [-u X] [-n N] [--looper-config LOOPER_CONFIG]
-                    [-S YAML [YAML ...]] [-P YAML [YAML ...]] [-l N] [-k N]
-                    [--sel-attr ATTR] [--sel-excl [E ...] | --sel-incl [I ...]]
-                    [--sel-flag [SELFLAG ...]] [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
+                    [-s S] [-c K [K ...]] [-u X] [-n N] [-j J]
+                    [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
+                    [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
+                    [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
+                    [--exc-flag [EXCFLAG ...]] [-a A [A ...]]
                     [config_file]
 
 Resubmit sample jobs with failed flags.
@@ -190,8 +194,11 @@ options:
   -x S, --command-extra S            String to append to every command
   -y S, --command-extra-override S   Same as command-extra, but overrides values in PEP
   -f, --skip-file-checks             Do not perform input file checks
-  -u X, --lump X                     Total input file size (GB) to batch into one job
-  -n N, --lumpn N                    Number of commands to batch into one job
+  -u X, --lump-s X                   Lump by size: total input file size (GB) to batch
+                                     into one job
+  -n N, --lump-n N                   Lump by number: number of samples to batch into one
+                                     job
+  -j J, --lump-j J                   Lump samples into number of jobs.
   --looper-config LOOPER_CONFIG      Looper configuration file (YAML)
   -S YAML [YAML ...], --sample-pipeline-interfaces YAML [YAML ...]
                                      Path to looper sample config file
@@ -225,7 +232,7 @@ sample selection arguments:
 usage: looper report [-h] [--looper-config LOOPER_CONFIG] [-S YAML [YAML ...]]
                      [-P YAML [YAML ...]] [-l N] [-k N] [--sel-attr ATTR]
                      [--sel-excl [E ...] | --sel-incl [I ...]] [--sel-flag [SELFLAG ...]]
-                     [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project]
+                     [--exc-flag [EXCFLAG ...]] [-a A [A ...]] [--project] [--portable]
                      [config_file]
 
 Create browsable HTML report of project results.
@@ -243,6 +250,7 @@ options:
                                      Path to looper project config file
   -a A [A ...], --amend A [A ...]    List of amendments to activate
   --project                          Process project-level pipelines
+  --portable                         Makes html report portable.
 
 sample selection arguments:
   Specify samples to include or exclude based on sample attribute values

looper/_version.py

@@ -1 +1 @@
-__version__ = "1.6.0"
+__version__ = "1.7.0"

looper/cli_looper.py

@@ -219,19 +219,27 @@ def add_subparser(cmd):
         for subparser in [run_subparser, rerun_subparser]:
             subparser.add_argument(
                 "-u",
-                "--lump",
+                "--lump-s",
                 default=None,
                 metavar="X",
                 type=html_range(min_val=0, max_val=100, step=0.1, value=0),
-                help="Total input file size (GB) to batch into one job",
+                help="Lump by size: total input file size (GB) to batch into one job",
             )
             subparser.add_argument(
                 "-n",
-                "--lumpn",
+                "--lump-n",
                 default=None,
                 metavar="N",
                 type=html_range(min_val=1, max_val="num_samples", value=1),
-                help="Number of commands to batch into one job",
+                help="Lump by number: number of samples to batch into one job",
+            )
+            subparser.add_argument(
+                "-j",
+                "--lump-j",
+                default=None,
+                metavar="J",
+                type=int,
+                help="Lump samples into number of jobs.",
             )
 
         check_subparser.add_argument(
@@ -501,6 +509,12 @@ def add_subparser(cmd):
             version="{}".format(" ".join(subparsers.choices.keys())),
         )
 
+        report_subparser.add_argument(
+            "--portable",
+            help="Makes html report portable.",
+            action="store_true",
+        )
+
         result.append(parser)
     return result
 

looper/conductor.py

@@ -6,6 +6,7 @@
 import subprocess
 import time
 import yaml
+from math import ceil
 from copy import copy, deepcopy
 from json import loads
 from subprocess import check_output
@@ -132,6 +133,7 @@ def __init__(
         compute_variables=None,
         max_cmds=None,
         max_size=None,
+        max_jobs=None,
         automatic=True,
         collate=False,
     ):
@@ -166,6 +168,8 @@ def __init__(
             include in a single job script.
         :param int | float | NoneType max_size: Upper bound on total file
             size of inputs used by the commands lumped into single job script.
+        :param int | float | NoneType max_jobs: Upper bound on total number of jobs to
+            group samples for submission.
         :param bool automatic: Whether the submission should be automatic once
             the pool reaches capacity.
         :param bool collate: Whether a collate job is to be submitted (runs on
@@ -200,6 +204,16 @@ def __init__(
                 "{}".format(self.extra_pipe_args)
             )
 
+        if max_jobs:
+            if max_jobs == 0 or max_jobs < 0:
+                raise ValueError(
+                    "If specified, max job command count must be a positive integer, greater than zero."
+                )
+
+            num_samples = len(self.prj.samples)
+            samples_per_job = num_samples / max_jobs
+            max_cmds = ceil(samples_per_job)
+
         if not self.collate:
             self.automatic = automatic
             if max_cmds is None and max_size is None:

looper/looper.py

@@ -404,8 +404,9 @@ def __call__(self, args, rerun=False, **compute_kwargs):
                 extra_args=args.command_extra,
                 extra_args_override=args.command_extra_override,
                 ignore_flags=args.ignore_flags,
-                max_cmds=args.lumpn,
-                max_size=args.lump,
+                max_cmds=args.lump_n,
+                max_size=args.lump_s,
+                max_jobs=args.lump_j,
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 
@@ -547,12 +548,16 @@ def __call__(self, args):
         p = self.prj
         project_level = args.project
 
+        portable = args.portable
+
         if project_level:
             psms = self.prj.get_pipestat_managers(project_level=True)
             print(psms)
             for name, psm in psms.items():
                 # Summarize will generate the static HTML Report Function
-                report_directory = psm.summarize(looper_samples=self.prj.samples)
+                report_directory = psm.summarize(
+                    looper_samples=self.prj.samples, portable=portable
+                )
                 print(f"Report directory: {report_directory}")
         else:
             for piface_source_samples in self.prj._samples_by_piface(
@@ -567,7 +572,9 @@ def __call__(self, args):
                 print(psms)
                 for name, psm in psms.items():
                     # Summarize will generate the static HTML Report Function
-                    report_directory = psm.summarize(looper_samples=self.prj.samples)
+                    report_directory = psm.summarize(
+                        looper_samples=self.prj.samples, portable=portable
+                    )
                     print(f"Report directory: {report_directory}")
 
 

requirements/requirements-all.txt

@@ -6,7 +6,7 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.1.2
 peppy>=0.40.0
-pipestat>=0.6.0
+pipestat>=0.8.0
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2

requirements/requirements-test.txt

@@ -1,4 +1,4 @@
-hypothesis
+hypothesis >= 6.84.3
 mock
 pytest
 pytest-cov

tests/smoketests/test_run.py

@@ -439,14 +439,31 @@ def test_looper_run_produces_submission_scripts(self, prep_temp_pep):
 
     def test_looper_lumping(self, prep_temp_pep):
         tp = prep_temp_pep
-        x = test_args_expansion(tp, "run", ["--lumpn", "2"])
+        x = test_args_expansion(tp, "run", ["--lump-n", "2"])
         try:
             main(test_args=x)
         except Exception:
             raise pytest.fail("DID RAISE {0}".format(Exception))
         sd = os.path.join(get_outdir(tp), "submission")
         verify_filecount_in_dir(sd, ".sub", 4)
 
+    def test_looper_lumping_jobs(self, prep_temp_pep):
+        tp = prep_temp_pep
+        x = test_args_expansion(tp, "run", ["--lump-j", "1"])
+        try:
+            main(test_args=x)
+        except Exception:
+            raise pytest.fail("DID RAISE {0}".format(Exception))
+        sd = os.path.join(get_outdir(tp), "submission")
+        verify_filecount_in_dir(sd, ".sub", 2)
+
+    def test_looper_lumping_jobs_negative(self, prep_temp_pep):
+        tp = prep_temp_pep
+        x = test_args_expansion(tp, "run", ["--lump-j", "-1"])
+
+        with pytest.raises(ValueError):
+            main(test_args=x)
+
     def test_looper_limiting(self, prep_temp_pep):
         tp = prep_temp_pep
         x = test_args_expansion(tp, "run", ["--limit", "2"])