From 101854dfa83cf4872b213afb64be698fb603a2de Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:48:22 -0400 Subject: [PATCH 1/4] first pass pulling bulker info from piface #211 --- looper/conductor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/looper/conductor.py b/looper/conductor.py index 4e23d0b3..ea949bba 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -685,6 +685,13 @@ def write_script(self, pool, size): else EXTRA_SAMPLE_CMD_TEMPLATE ) templ += extras_template + if ( + "bulker" in self.pl_iface.exp + and "bulker_crate" in self.pl_iface.exp["bulker"] + ): + _LOGGER.warning(self.pl_iface.exp["bulker"]["bulker_crate"]) + templ = f"bulker run {self.pl_iface.exp['bulker']['bulker_crate']} " + templ + for sample in pool: # cascading compute settings determination: # divcfg < pipeline interface < config < CLI From 25a100b6a26efd13deb7af64fabed6ff333596bb Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Thu, 13 Jun 2024 15:18:37 -0400 Subject: [PATCH 2/4] remove duplicated divvy_templates to avoid confusion --- divvy_templates/localhost_bulker_template.sub | 10 ---------- divvy_templates/localhost_docker_template.sub | 8 -------- .../localhost_singularity_template.sub | 9 --------- divvy_templates/localhost_template.sub | 8 -------- divvy_templates/lsf_template.sub | 4 ---- divvy_templates/sge_template.sub | 1 - divvy_templates/slurm_singularity_template.sub | 17 ----------------- divvy_templates/slurm_template.sub | 14 -------------- 8 files changed, 71 deletions(-) delete mode 100755 divvy_templates/localhost_bulker_template.sub delete mode 100755 divvy_templates/localhost_docker_template.sub delete mode 100755 divvy_templates/localhost_singularity_template.sub delete mode 100755 divvy_templates/localhost_template.sub delete mode 100755 divvy_templates/lsf_template.sub delete mode 100755 divvy_templates/sge_template.sub delete mode 100755 divvy_templates/slurm_singularity_template.sub delete mode 100755 divvy_templates/slurm_template.sub diff --git a/divvy_templates/localhost_bulker_template.sub b/divvy_templates/localhost_bulker_template.sub deleted file mode 100755 index 8ef7e7e3..00000000 --- a/divvy_templates/localhost_bulker_template.sub +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -eval "$(bulker activate -e {BULKER_CRATE})" - -{ - {CODE} -} | tee {LOGFILE} -i diff --git a/divvy_templates/localhost_docker_template.sub b/divvy_templates/localhost_docker_template.sub deleted file mode 100755 index a00ad037..00000000 --- a/divvy_templates/localhost_docker_template.sub +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -docker run --rm -it {DOCKER_ARGS} {DOCKER_IMAGE} {CODE} -} | tee {LOGFILE} --ignore-interrupts diff --git a/divvy_templates/localhost_singularity_template.sub b/divvy_templates/localhost_singularity_template.sub deleted file mode 100755 index fdb1e486..00000000 --- a/divvy_templates/localhost_singularity_template.sub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -singularity exec instance://{JOBNAME}_image {CODE} -} | tee {LOGFILE} --ignore-interrupts diff --git a/divvy_templates/localhost_template.sub b/divvy_templates/localhost_template.sub deleted file mode 100755 index 9b64b754..00000000 --- a/divvy_templates/localhost_template.sub +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -{CODE} -} | tee {LOGFILE} diff --git a/divvy_templates/lsf_template.sub b/divvy_templates/lsf_template.sub deleted file mode 100755 index ac2bfc1e..00000000 --- a/divvy_templates/lsf_template.sub +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -bsub -n{CORES} -W {TIME} -R \"rusage[mem={MEM}]\" -o {LOGFILE} {CODE} - diff --git a/divvy_templates/sge_template.sub b/divvy_templates/sge_template.sub deleted file mode 100755 index 0964f4df..00000000 --- a/divvy_templates/sge_template.sub +++ /dev/null @@ -1 +0,0 @@ -This has not been implemented, but you could add whatever cluster submission systems here, just use the slurm_template as an example. diff --git a/divvy_templates/slurm_singularity_template.sub b/divvy_templates/slurm_singularity_template.sub deleted file mode 100755 index eb8359be..00000000 --- a/divvy_templates/slurm_singularity_template.sub +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -#SBATCH --job-name='{JOBNAME}' -#SBATCH --output='{LOGFILE}' -#SBATCH --mem='{MEM}' -#SBATCH --cpus-per-task='{CORES}' -#SBATCH --time='{TIME}' -#SBATCH --partition='{PARTITION}' -#SBATCH -m block -#SBATCH --ntasks=1 - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -srun singularity exec instance://{JOBNAME}_image {CODE} - -singularity instance.stop {JOBNAME}_image diff --git a/divvy_templates/slurm_template.sub b/divvy_templates/slurm_template.sub deleted file mode 100755 index f9d74564..00000000 --- a/divvy_templates/slurm_template.sub +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -#SBATCH --job-name='{JOBNAME}' -#SBATCH --output='{LOGFILE}' -#SBATCH --mem='{MEM}' -#SBATCH --cpus-per-task='{CORES}' -#SBATCH --time='{TIME}' -#SBATCH --partition='{PARTITION}' -#SBATCH -m block -#SBATCH --ntasks=1 - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{CODE} From 7ca8ee68ea1e0db59db8dc1534e250af6e1c4c2e Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Thu, 13 Jun 2024 19:02:09 -0400 Subject: [PATCH 3/4] add --bulker flag, load crate for every piface, add running crate to the command if applicable --- looper/command_models/arguments.py | 6 ++++++ looper/command_models/commands.py | 1 + looper/conductor.py | 27 +++++++++++++++++++++------ looper/looper.py | 1 + 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/looper/command_models/arguments.py b/looper/command_models/arguments.py index 8c484d33..7fe640f5 100644 --- a/looper/command_models/arguments.py +++ b/looper/command_models/arguments.py @@ -281,3 +281,9 @@ class ArgumentEnum(enum.Enum): default=(bool, False), description="Is this command executed for project-level?", ) + + BULKER = Argument( + name="bulker", + default=(bool, False), + description="Run bulker", + ) diff --git a/looper/command_models/commands.py b/looper/command_models/commands.py index 233cfd0b..932261d9 100644 --- a/looper/command_models/commands.py +++ b/looper/command_models/commands.py @@ -79,6 +79,7 @@ def create_model(self) -> Type[pydantic.BaseModel]: ArgumentEnum.SKIP_FILE_CHECKS.value, ArgumentEnum.COMPUTE.value, ArgumentEnum.PACKAGE.value, + ArgumentEnum.BULKER.value, ], ) diff --git a/looper/conductor.py b/looper/conductor.py index ea949bba..bf2e718b 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -151,6 +151,7 @@ def __init__( max_jobs=None, automatic=True, collate=False, + bulker=False, ): """ Create a job submission manager. @@ -214,6 +215,22 @@ def __init__( self._failed_sample_names = [] self._curr_skip_pool = [] self.process_id = None # this is used for currently submitted subprocess + self.bulker = bulker + self.bulker_commands = None + + if bulker: + if "bulker_crate" in self.pl_iface.exp: + crates = [] + self.bulker_commands = [] + if isinstance(self.pl_iface.exp["bulker_crate"], str): + crates = [self.pl_iface.exp["bulker_crate"]] + elif isinstance(self.pl_iface.exp["bulker_crate"], list): + crates = self.pl_iface.exp["bulker_crate"] + for crate in crates: + bulker_load_cmd = f"bulker load -b -f {crate}" + bulker_load_process = subprocess.Popen(bulker_load_cmd, shell=True) + bulker_load_process.wait() + self.bulker_commands.append(f"bulker run {crate} ") if self.extra_pipe_args: _LOGGER.debug( @@ -685,12 +702,10 @@ def write_script(self, pool, size): else EXTRA_SAMPLE_CMD_TEMPLATE ) templ += extras_template - if ( - "bulker" in self.pl_iface.exp - and "bulker_crate" in self.pl_iface.exp["bulker"] - ): - _LOGGER.warning(self.pl_iface.exp["bulker"]["bulker_crate"]) - templ = f"bulker run {self.pl_iface.exp['bulker']['bulker_crate']} " + templ + + if self.bulker_commands: + for bulker_command in self.bulker_commands: + templ = bulker_command + templ for sample in pool: # cascading compute settings determination: diff --git a/looper/looper.py b/looper/looper.py index 1eea6edd..66534086 100755 --- a/looper/looper.py +++ b/looper/looper.py @@ -419,6 +419,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs): max_cmds=getattr(args, "lump_n", None), max_size=getattr(args, "lump", None), max_jobs=getattr(args, "lump_j", None), + bulker=getattr(args, "bulker", None), ) submission_conductors[piface.pipe_iface_file] = conductor From 1a474f6e0007ecee198e572710df1c4310107595 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:08:53 -0400 Subject: [PATCH 4/4] remove appending commands --- looper/conductor.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/looper/conductor.py b/looper/conductor.py index bf2e718b..ee4758f3 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -216,7 +216,6 @@ def __init__( self._curr_skip_pool = [] self.process_id = None # this is used for currently submitted subprocess self.bulker = bulker - self.bulker_commands = None if bulker: if "bulker_crate" in self.pl_iface.exp: @@ -230,7 +229,6 @@ def __init__( bulker_load_cmd = f"bulker load -b -f {crate}" bulker_load_process = subprocess.Popen(bulker_load_cmd, shell=True) bulker_load_process.wait() - self.bulker_commands.append(f"bulker run {crate} ") if self.extra_pipe_args: _LOGGER.debug( @@ -703,10 +701,6 @@ def write_script(self, pool, size): ) templ += extras_template - if self.bulker_commands: - for bulker_command in self.bulker_commands: - templ = bulker_command + templ - for sample in pool: # cascading compute settings determination: # divcfg < pipeline interface < config < CLI