Use drmaa instead of running qsub/sbatch commands

bidscoin/bidsapps/slicereport.py

@@ -111,24 +111,31 @@ def slicer_append(inputimage: Path, operations: str, outlineimage: Path, mainopt
                f"mv {montage.name} {montage.parent}\n" \
                + (f"rm -r {workdir}" if not DEBUG else '')
 
-    # Wrap the shell command with a cluster submit command
-    mem = mem or ('10' if inputimage.stat().st_size > 50 * 1024**2 else '2')  # Ask for more resources if we have a large (e.g. > 50 MB / 4D) input image
-    outpath = workdir if DEBUG else tempfile.gettempdir()
-    if cluster == 'torque':
-        command = f"qsub -l walltime=0:02:00,mem={mem}gb -N slicereport -j oe -o {outpath} << EOF\n#!/bin/bash\n{command}\nEOF"
-    elif cluster == 'slurm':
-        command = f"sbatch --time=0:02:00 --mem={mem}G --job-name slicereport -o {outpath}/slurm-%x-%j.out << EOF\n#!/bin/bash\n{command}\nEOF"
-    elif cluster:
-        LOGGER.error(f"Invalid cluster manager `{cluster}`")
-        exit(1)
-
-    # Run the command
-    LOGGER.bcdebug(f"Command: {command}")
-    process = subprocess.run(command, shell=True, capture_output=True, text=True)
-    if process.stderr or process.returncode != 0:
-        LOGGER.warning(f"{command}\nErrorcode {process.returncode}:\n{process.stdout}\n{process.stderr}")
-    if process.returncode != 0:
-        sys.exit(process.returncode)
+    # Run the command on the HPC cluster or directly in the shell
+    if cluster:
+        from drmaa import Session as drmaasession   # Lazy import to avoid import error on non-HPC systems
+
+        script = workdir/'slicereport.sh'
+        script.write_text('#!/bin/bash\n' + command)
+        script.chmod(0o744)
+        with drmaasession() as pbatch:
+            jt                     = pbatch.createJobTemplate()
+            jt.jobEnvironment      = os.environ
+            jt.remoteCommand       = script
+            jt.nativeSpecification = cluster
+            jt.joinFiles           = True
+            jt.jobName             = 'slicereport'
+            jt.outputPath          = f"{os.getenv('HOSTNAME')}:{workdir}/{jt.jobName}.out"
+            jobid                  = pbatch.runJob(jt)
+            LOGGER.info(f"Your slicereport job has been submitted with ID: {jobid}")
+
+    else:
+        LOGGER.bcdebug(f"Command: {command}")
+        process = subprocess.run(command, shell=True, capture_output=True, text=True)
+        if process.stderr or process.returncode != 0:
+            LOGGER.warning(f"{command}\nErrorcode {process.returncode}:\n{process.stdout}\n{process.stderr}")
+        if process.returncode != 0:
+            sys.exit(process.returncode)
 
 
 def slicereport(bidsfolder: str, pattern: str, outlinepattern: str, outlineimage: str, participant: list, reportfolder: str, xlinkfolder: str, qcscores: list, cluster: str, mem: str, operations: str, suboperations: str, options: list, outputs: list, suboptions: list, suboutputs: list):

bidscoin/cli/_slicereport.py

@@ -43,7 +43,7 @@ def get_parser():
 examples:
   slicereport bids anat/*_T1w*
   slicereport bids anat/*_T2w* -r QC/slicereport_T2 -x QC/slicereport_T1
-  slicereport bids fmap/*_phasediff* -o fmap/*_magnitude1*
+  slicereport bids fmap/*_phasediff* -o fmap/*_magnitude1* -c "--time=00:10:00 --mem=2000"
   slicereport bids/derivatives/fmriprep func/*desc-preproc_bold* --suboperations " -Tstd"
   slicereport bids/derivatives/fmriprep anat/*desc-preproc_T1w* -o anat/*label-GM* -x bids/derivatives/fmriprep
   slicereport bids/derivatives/deface anat/*_T1w* -o bids:anat/*_T1w* --options L e 0.05
@@ -60,8 +60,7 @@ def get_parser():
     parser.add_argument('-r','--reportfolder',   help="The folder where the report is saved (default: bidsfolder/derivatives/slicereport)", metavar='FOLDER')
     parser.add_argument('-x','--xlinkfolder',    help="A (list of) QC report folder(s) with cross-linkable sub-reports, e.g. bidsfolder/derivatives/mriqc", nargs='+', metavar='FOLDER')
     parser.add_argument('-q','--qcscores',       help="Column names for creating an accompanying tsv-file to store QC-rating scores (default: rating_overall)", default=['rating_overall'], nargs='+', metavar='NAME')
-    parser.add_argument('-c','--cluster',        help='Use `torque` or `slurm` to submit the slicereport jobs to a high-performance compute (HPC) cluster', choices=['torque','slurm'])
-    parser.add_argument('-m','--mem',            help='The amount of requested memory in GB for the cluster jobs', default='', metavar='GB')
+    parser.add_argument('-c','--cluster',        help='Use the DRMAA library to submit the slicereport jobs to a high-performance compute (HPC) cluster. You can add an opaque DRMAA argument with native specifications for your HPC resource manager (NB: Use quotes and include at least one space character to prevent premature parsing -- see examples)', metavar='SPECS', nargs='?', const='-l walltime=00:10:00,mem=2gb', type=str)
     parser.add_argument('--operations',          help='One or more fslmaths operations that are performed on the input image (before slicing it for the report). OPERATIONS is opaquely passed as is: `fslmaths inputimage OPERATIONS reportimage`. NB: Use quotes and include at least one space character to prevent premature parsing, e.g. " -Tmean" or "-Tstd -s 3" (default: -Tmean)', default='-Tmean')
     parser.add_argument('--suboperations',       help='The same as OPERATIONS but then for the sub-report instead of the main report: `fslmaths inputimage SUBOPERATIONS subreportimage` (default: -Tmean)', default='-Tmean', metavar='OPERATIONS')
     parser.add_argument('--options',             help='Main options of slicer (see below). (default: "s 1")', default=['s','1'], nargs='+')