Merge pull request #74 from pnlbwh/redhat-9

Upgrade to Redhat 9 OS

docs/README.md

@@ -99,16 +99,26 @@ Finally, see [here](#running-luigi-pnlpipe) for instructions about running the p
 
 ## luigi package
 
-A *client* (PNL external collaborator) can use the official luigi package installed in `pnlpipe3` conda environment.
-However, a *server* should install Tashrif's development for `/history/by_task_id/` URL to function:
+Tashrif's development of `/history/by_task_id/` feature got merged in the official `luigi` package.
+Until a release is available, you should install it as:
 
-    pip install git+https://github.com/tashrifbillah/luigi.git@89c9aa750de8ae2badabe435d98c02e44a1aa8b4
+    pip install git+https://github.com/spotify/luigi.git@172128c3de7a41411a10e61e3c675b76595793e2
 
-*luigi-pnlpipe* itself will not fail without Tashrif's development on the *server* side. That means, you can 
-also use the official luigi package on the *server* side. But you will not be able to redirect to 
-`/history/by_task_id/` URLs generated in `*.log.html` provenance files. Notably, the provenance files 
-are generated on the *client* side by [_provenance.py](https://github.com/pnlbwh/luigi-pnlpipe/blob/afa6c8a86d481d8fe5d04ba1ceb533b5da740c32/workflows/_provenance.py) when *luigi-pipeline* is run.
+However, `luigi` history feature does not work with the latest `sqlalchemy`. Install an older version as:
 
+    pip install sqlalchemy==1.4.54
+
+Reference: https://github.com/spotify/luigi/issues/3320
+
+If you are an outside collaborator, you can launch a Luigi server as:
+
+    luigi-pnlpipe/start_server.sh
+
+And subsequently, uncomment this line in `luigi-pnlpipe/luigi.cfg` to use it:
+
+    default-scheduler-url = http://localhost:8082/
+
+Psychiatry Neuroimaging Laboratory staff have it available readily. They do not need to launch one.
 
 ## CNN-Diffusion-MRIBrain-Segmentation
 

luigi.cfg

@@ -1,10 +1,13 @@
 [core]
+# uncomment one of the Luigi servers
+
 # for individual machine
 # default-scheduler-url = http://localhost:8082/
-# for High Performance Cluster (lsf script, bsub command)
-# default-scheduler-url = http://cmu166.research.partners.org:8082/
+# PNL hosted private server
+# default-scheduler-url = http://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnl-elite-1.partners.org/
 # PNL hosted public server
-default-scheduler-url = https://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnlservers.bwh.harvard.edu/luigi/
+# default-scheduler-url = https://${LUIGI_USERNAME}:${LUIGI_PASSWORD}@pnlservers.bwh.harvard.edu/luigi/
+
 log_level = INFO
 
 [scheduler]

luigid.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=PNL Luigi server at http://pnl-elite-1.partners.org/
+After=network-online.target nginx.service
+Wants=network-online.target nginx.service
+
+[Service]
+User=sf284
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/opt/start_server.sh
+ExecStop=pkill luigid
+KillMode=process
+TimeoutSec=60
+
+[Install]
+WantedBy=multi-user.target
+

params/cte/dwi_pipe_params.cfg

@@ -21,7 +21,7 @@ filter:
 acqp: /data/pnl/DIAGNOSE_CTE_U01/acqp.txt
 index: /data/pnl/DIAGNOSE_CTE_U01/index.txt
 config: /data/pnl/DIAGNOSE_CTE_U01/eddy_config.txt
-useGpu: False
+useGpu: True
 FslOutDir: fsl_eddy
 
 
@@ -59,6 +59,8 @@ wma_cleanup: 0
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
 [PnlEddy]

params/dwi_pipe_params.cfg

@@ -78,8 +78,12 @@ wma_cleanup: 0
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
+[SynB0]
+
 [PnlEddy]
 
 [FslEddy]

params/hcp/dwi_pipe_params.cfg

@@ -34,22 +34,22 @@ HcpOutDir: hcppipe
 
 
 ## [GibbsUn] ##
-unring_nproc: 4
+unring_nproc: 8
 
 
 ## [Ukf] ##
-ukf_params:
+ukf_params: --seedsPerVoxel,1
 eddy_epi_task: HcpPipe
 bhigh: 2000
 
 
 ## [WMA800] ##
-slicer_exec: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh
-FiberTractMeasurements: /data/pnl/soft/pnlpipe3/Slicer-4.10.2-linux-amd64/SlicerWithExtensions.sh --launch FiberTractMeasurements
-atlas: /data/pnl/soft/pnlpipe3/ORG-Atlases-1.2
+slicer_exec: /software/rocky9/Slicer-5.6.1-linux-amd64/Slicer
+FiberTractMeasurements: /software/rocky9/Slicer-5.6.1-linux-amd64/slicer.org/Extensions-32438/SlicerDMRI/lib/Slicer-5.6/cli-modules/FiberTractMeasurements
+atlas: /software/rocky9/ORG-Atlases-1.2
 wma_nproc: 4
 xvfb: 1
-wma_cleanup: 0
+wma_cleanup: 2
 
 
 [StructMask]
@@ -58,6 +58,8 @@ wma_cleanup: 0
 
 [BseMask]
 
+[GibbsUn]
+
 [CnnMask]
 
 [PnlEddy]

params/synb0/acqparams.txt

@@ -0,0 +1,2 @@
+0 1 0 0.05
+0 1 0 0

params/synb0/dwi_pipe_params.cfg

@@ -0,0 +1,75 @@
+[DEFAULT]
+
+## [StructMask] ##
+mask_method: hd-bet
+mask_qc: False
+
+
+## [StructMask] [PnlEddy] [EddyEpi] ##
+debug: False
+
+
+## [CnnMask] ##
+model_folder: /software/rocky9/CNN-Diffusion-MRIBrain-Segmentation/model_folder
+percentile: 97
+filter:
+
+
+## [FslEddy] [TopupEddy] ##
+acqp: /software/rocky9/luigi-tutorial/edcrp/acqparams.txt
+index: /software/rocky9/luigi-tutorial/edcrp/index.txt
+config: /software/rocky9/pnlNipype/scripts/eddy_config.txt
+useGpu: False
+FslOutDir: fsl_eddy
+
+
+## [EddyEpi] ##
+epi_nproc: 8
+eddy_task: FslEddy
+
+
+## [HcpPipe] ##
+HcpOutDir: hcppipe
+
+
+## [GibbsUn] ##
+unring_nproc: 4
+
+
+## [Ukf] ##
+ukf_params: --numThreads,8
+eddy_epi_task: HcpPipe
+bhigh: 2000
+
+
+## [WMA800] ##
+slicer_exec: /software/rocky9/Slicer-5.6.1-linux-amd64/Slicer
+FiberTractMeasurements: /software/rocky9/Slicer-5.6.1-linux-amd64/slicer.org/Extensions-32438/SlicerDMRI/lib/Slicer-5.6/cli-modules/FiberTractMeasurements
+atlas: /data/pnl/soft/pnlpipe3/ORG-Atlases-1.2
+wma_nproc: 4
+xvfb: 1
+wma_cleanup: 
+
+[StructMask]
+
+[BseExtract]
+
+[CnnMask]
+
+[GibbsUn]
+
+[SynB0]
+
+[PnlEddy]
+
+[FslEddy]
+
+[TopupEddy]
+
+[EddyEpi]
+
+[HcpPipe]
+
+[Ukf]
+
+[Wma800]

start_server.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /opt/
+miniconda3/bin/luigid --background --logdir luigi-server

workflows/ExecuteTask.py

@@ -2,7 +2,7 @@
 
 import argparse
 from conversion import read_cases
-from luigi import build, configuration
+from luigi import build, configuration, parameter
 from _define_outputs import IO
 from struct_pipe import StructMask, Freesurfer
 from dwi_pipe import DwiAlign, GibbsUn, CnnMask, \
@@ -13,6 +13,8 @@
 from tempfile import gettempdir
 from glob import glob
 
+import warnings
+warnings.filterwarnings(action='ignore', category=parameter.UnconsumedParameterWarning)
 
 def _rm_tempfiles(names):
 

workflows/_deps_tree.py

@@ -61,7 +61,7 @@ def get_record_id(task_id):
     # useful debug commands
     # full path necessary after .open
     # sqlite3
-    #   sqlite> .open /home/tb571/luigi-task-hist.db
+    #   sqlite> .open /path/to/luigi-task-hist.db
     #   sqlite> SELECT * FROM tasks;
     #   sqlite> .quit
     cur = conn.cursor()

workflows/_provenance.py

@@ -1,6 +1,6 @@
 from _deps_tree import print_tree, print_history_tree
 from os.path import join as pjoin, dirname, isfile
-from os import getpid, environ
+from os import getpid
 from subprocess import check_call, check_output
 from tempfile import gettempdir
 
@@ -23,7 +23,7 @@ def _get_env():
 
     # read hashes
     with open(hash_file) as f:
-        content= f.read().split()
+        content= f.read().strip().split('\n')
 
     # save hashes in a dictionary for integrating with json provenance
     hash_dict={}
@@ -34,7 +34,7 @@ def _get_env():
     # export conda env
     env_file= pjoin(gettempdir(), f'env-{getpid()}.yml')
     if not isfile(env_file):
-        check_output(f"{environ['CONDA_EXE']} env export > {env_file}", shell=True)
+        check_output(f"conda env export > {env_file}", shell=True)
 
     with open(env_file) as f:
         hash_dict['conda_env']= f.read()
@@ -66,7 +66,7 @@ def write_provenance(obj, output=None):
 
     logfile= output.dirname.join(output.stem)+'.log.html'
     with open(logfile,'w') as f:
-        template= template.replace('{{output}}',output.basename)
+        template= template.replace('{{output}}',output.name)
         template= template.replace('{{textHistory}}',tree)
         template= template.replace('{{htmlHistory}}',history_tree)
         f.write(template)

workflows/_synb0_eddy.sh

@@ -32,7 +32,7 @@ if [ ! -f OUTPUTS/b0_all_topup.nii.gz ]
 then
     TMPDIR=$TMPDIR \
     singularity run -e -B INPUTS/:/INPUTS -B OUTPUTS/:/OUTPUTS \
-    -B ${NEW_SOFT_DIR}/fs7.1.0/license.txt:/extra/freesurfer/license.txt \
+    -B ${FREESURFER_HOME}/license.txt:/extra/freesurfer/license.txt \
     ${NEW_SOFT_DIR}/containers/synb0-disco_v3.0.sif --stripped
 fi