Add bulker support

looper/command_models/arguments.py

@@ -281,3 +281,9 @@ class ArgumentEnum(enum.Enum):
         default=(bool, False),
         description="Is this command executed for project-level?",
     )
+
+    BULKER = Argument(
+        name="bulker",
+        default=(bool, False),
+        description="Run bulker",
+    )

looper/command_models/commands.py

@@ -79,6 +79,7 @@ def create_model(self) -> Type[pydantic.BaseModel]:
         ArgumentEnum.SKIP_FILE_CHECKS.value,
         ArgumentEnum.COMPUTE.value,
         ArgumentEnum.PACKAGE.value,
+        ArgumentEnum.BULKER.value,
     ],
 )
 

looper/conductor.py

@@ -151,6 +151,7 @@ def __init__(
         max_jobs=None,
         automatic=True,
         collate=False,
+        bulker=False,
     ):
         """
         Create a job submission manager.
@@ -214,6 +215,20 @@ def __init__(
         self._failed_sample_names = []
         self._curr_skip_pool = []
         self.process_id = None  # this is used for currently submitted subprocess
+        self.bulker = bulker
+
+        if bulker:
+            if "bulker_crate" in self.pl_iface.exp:
+                crates = []
+                self.bulker_commands = []
+                if isinstance(self.pl_iface.exp["bulker_crate"], str):
+                    crates = [self.pl_iface.exp["bulker_crate"]]
+                elif isinstance(self.pl_iface.exp["bulker_crate"], list):
+                    crates = self.pl_iface.exp["bulker_crate"]
+                for crate in crates:
+                    bulker_load_cmd = f"bulker load -b -f {crate}"
+                    bulker_load_process = subprocess.Popen(bulker_load_cmd, shell=True)
+                    bulker_load_process.wait()
 
         if self.extra_pipe_args:
             _LOGGER.debug(
@@ -685,6 +700,7 @@ def write_script(self, pool, size):
                 else EXTRA_SAMPLE_CMD_TEMPLATE
             )
             templ += extras_template
+
         for sample in pool:
             # cascading compute settings determination:
             # divcfg < pipeline interface < config <  CLI

looper/looper.py

@@ -419,6 +419,7 @@ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
                 max_cmds=getattr(args, "lump_n", None),
                 max_size=getattr(args, "lump", None),
                 max_jobs=getattr(args, "lump_j", None),
+                bulker=getattr(args, "bulker", None),
             )
             submission_conductors[piface.pipe_iface_file] = conductor