From 49d4ffba57e7614f6d06e41b9f5b3046357a0b1d Mon Sep 17 00:00:00 2001 From: Austin Hoag Date: Thu, 18 Nov 2021 13:02:48 -0500 Subject: [PATCH] use brainpipe conda environment --- ...stitcher.sh => example_tracing_pipeline.sh | 22 ++++++--- registration_pipeline.sh | 25 ---------- slurm_files/cnn_postprocess.sh | 4 +- slurm_files/cnn_preprocess.sh | 4 +- slurm_files/cnn_step0.sh | 4 +- slurm_files/cnn_step1.sh | 4 +- slurm_files/cnn_step1_check.sh | 4 +- slurm_files/cnn_step2.sh | 4 +- slurm_files/cnn_step21.sh | 4 +- slurm_files/cnn_step3.sh | 4 +- slurm_files/cnn_step4.sh | 4 +- sub_registration.sh | 46 ------------------- sub_update_registration.sh | 38 --------------- test_registration.sh | 46 ------------------- tools/utils/directorydeterminer.py | 2 - 15 files changed, 34 insertions(+), 181 deletions(-) rename sub_registration_terastitcher.sh => example_tracing_pipeline.sh (64%) delete mode 100644 registration_pipeline.sh delete mode 100644 sub_registration.sh delete mode 100644 sub_update_registration.sh delete mode 100644 test_registration.sh diff --git a/sub_registration_terastitcher.sh b/example_tracing_pipeline.sh similarity index 64% rename from sub_registration_terastitcher.sh rename to example_tracing_pipeline.sh index 3e13e8ea..df989ca8 100644 --- a/sub_registration_terastitcher.sh +++ b/example_tracing_pipeline.sh @@ -17,16 +17,14 @@ cat /proc/meminfo echo "Array Allocation Number: $SLURM_ARRAY_JOB_ID" echo "Array Index: $SLURM_ARRAY_TASK_ID" - -module load anacondapy/5.3.1 -module load elastix/4.8 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe #set up dictionary and save OUT0=$(sbatch --array=0 slurm_files/step0.sh) echo $OUT0 -#process zplns, check that 1000 > zplns/slurmfactor +#process zplns, assumes with tracing you are using terastitcher OUT1=$(sbatch --dependency=afterany:${OUT0##* } --array=0-2 slurm_files/step1.sh) echo $OUT1 @@ -34,6 +32,18 @@ echo $OUT1 OUT2=$(sbatch --dependency=afterany:${OUT1##* } --array=0-3 slurm_files/step2.sh) echo $OUT2 +#generate memmap array of full size cell channel data +OUT4=$(sbatch --dependency=afterany:${OUT1##* } slurm_files/cnn_step0.sh "`pwd`") +echo $OUT4 + +#generate chunks for cnn input +OUT5=$(sbatch --dependency=afterany:${OUT4##* } --array=0-130 slurm_files/cnn_step1.sh "`pwd`") +echo $OUT5 + +#check if correct number of patches were made +OUT6=$(sbatch --dependency=afterany:${OUT5##* } slurm_files/cnn_step1_check.sh "`pwd`") +echo $OUT6 + #run elastix OUT3=$(sbatch --dependency=afterany:${OUT2##* } --array=0-2 slurm_files/step3.sh) echo $OUT3 @@ -43,4 +53,4 @@ echo $OUT3 # after = go once the specified job starts # afterany = go if the specified job finishes, regardless of success # afternotok = go if the specified job fails -# afterok = go if the specified job completes successfully \ No newline at end of file +# afterok = go if the specified job completes successfully diff --git a/registration_pipeline.sh b/registration_pipeline.sh deleted file mode 100644 index af5911e1..00000000 --- a/registration_pipeline.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/env bash - - -#set up dictionary and save -OUT0=$(sbatch --array=0 slurm_files/step0.sh) -echo $OUT0 - -#process zplns, check that 1000 > zplns/slurmfactor -OUT1=$(sbatch --dependency=afterany:${OUT0##* } --array=0-26 slurm_files/step1.sh) -echo $OUT1 - -#combine stacks into single tifffiles -OUT2=$(sbatch --dependency=afterany:${OUT1##* } --array=0 slurm_files/step2.sh) -echo $OUT2 - -#run elastix -OUT3=$(sbatch --dependency=afterany:${OUT2##* } --array=0-2 slurm_files/step3.sh) -echo $OUT3 - - -# Usage notes: -# after = go once the specified job starts -# afterany = go if the specified job finishes, regardless of success -# afternotok = go if the specified job fails -# afterok = go if the specified job completes successfully diff --git a/slurm_files/cnn_postprocess.sh b/slurm_files/cnn_postprocess.sh index 113b47ba..075c039f 100644 --- a/slurm_files/cnn_postprocess.sh +++ b/slurm_files/cnn_postprocess.sh @@ -13,8 +13,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list cat /proc/meminfo -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe #generate memmap array of reconstructed cnn output OUT0=$(sbatch slurm_files/cnn_step21.sh "`pwd`") diff --git a/slurm_files/cnn_preprocess.sh b/slurm_files/cnn_preprocess.sh index 41288f99..f38c5c5e 100644 --- a/slurm_files/cnn_preprocess.sh +++ b/slurm_files/cnn_preprocess.sh @@ -13,8 +13,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list cat /proc/meminfo -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" diff --git a/slurm_files/cnn_step0.sh b/slurm_files/cnn_step0.sh index ef335549..4ba00602 100644 --- a/slurm_files/cnn_step0.sh +++ b/slurm_files/cnn_step0.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" diff --git a/slurm_files/cnn_step1.sh b/slurm_files/cnn_step1.sh index 49a4cdb0..41ee860d 100644 --- a/slurm_files/cnn_step1.sh +++ b/slurm_files/cnn_step1.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" echo "Array Index: $SLURM_ARRAY_TASK_ID" diff --git a/slurm_files/cnn_step1_check.sh b/slurm_files/cnn_step1_check.sh index a5fbcf82..4c3c8def 100644 --- a/slurm_files/cnn_step1_check.sh +++ b/slurm_files/cnn_step1_check.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" echo "Array Index: $SLURM_ARRAY_TASK_ID" diff --git a/slurm_files/cnn_step2.sh b/slurm_files/cnn_step2.sh index 9d3df99d..5b3560d3 100644 --- a/slurm_files/cnn_step2.sh +++ b/slurm_files/cnn_step2.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" diff --git a/slurm_files/cnn_step21.sh b/slurm_files/cnn_step21.sh index dba632ea..1a7a3e8d 100644 --- a/slurm_files/cnn_step21.sh +++ b/slurm_files/cnn_step21.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" diff --git a/slurm_files/cnn_step3.sh b/slurm_files/cnn_step3.sh index e6f5c356..2525505e 100644 --- a/slurm_files/cnn_step3.sh +++ b/slurm_files/cnn_step3.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" echo "Array Index: $SLURM_ARRAY_TASK_ID" diff --git a/slurm_files/cnn_step4.sh b/slurm_files/cnn_step4.sh index 978f6d75..5c7ee79c 100644 --- a/slurm_files/cnn_step4.sh +++ b/slurm_files/cnn_step4.sh @@ -14,8 +14,8 @@ echo "on host: `hostname` " cat /proc/$$/status | grep Cpus_allowed_list -module load anacondapy/5.3.1 -. activate lightsheet +module load anacondapy/2020.11 +. activate brainpipe echo "Experiment name:" "`pwd`" diff --git a/sub_registration.sh b/sub_registration.sh deleted file mode 100644 index 738a04ab..00000000 --- a/sub_registration.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/env bash -# -#SBATCH -p all # partition (queue) -#SBATCH -n 1 # number of cores -#SBATCH -t 20 # time (minutes) -#SBATCH -o logs/outmain_registration.out # STDOUT -#SBATCH -e logs/outmain_registration.err # STDERR - - -echo "In the directory: `pwd` " -echo "As the user: `whoami` " -echo "on host: `hostname` " - -cat /proc/$$/status | grep Cpus_allowed_list -cat /proc/meminfo - -echo "Array Allocation Number: $SLURM_ARRAY_JOB_ID" -echo "Array Index: $SLURM_ARRAY_TASK_ID" - - -module load anacondapy/5.3.1 -module load elastix/4.8 -. activate lightsheet - -#set up dictionary and save -OUT0=$(sbatch --array=0 slurm_files/step0.sh) -echo $OUT0 - -#process zplns, check that 1000 > zplns/slurmfactor -OUT1=$(sbatch --dependency=afterany:${OUT0##* } --array=0-80 slurm_files/step1.sh) -echo $OUT1 - -#combine stacks into single tifffiles -OUT2=$(sbatch --dependency=afterany:${OUT1##* } --array=0-3 slurm_files/step2.sh) -echo $OUT2 - -#run elastix -OUT3=$(sbatch --dependency=afterany:${OUT2##* } --array=0-2 slurm_files/step3.sh) -echo $OUT3 - - -# Usage notes: -# after = go once the specified job starts -# afterany = go if the specified job finishes, regardless of success -# afternotok = go if the specified job fails -# afterok = go if the specified job completes successfully diff --git a/sub_update_registration.sh b/sub_update_registration.sh deleted file mode 100644 index adae1cac..00000000 --- a/sub_update_registration.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/env bash -# -#SBATCH -p all # partition (queue) -#SBATCH -n 1 # number of cores -#SBATCH -t 20 # time (minutes) -#SBATCH -o logs/update_registration.out # STDOUT -#SBATCH -e logs/update_registration.err # STDERR - - -echo "In the directory: `pwd` " -echo "As the user: `whoami` " -echo "on host: `hostname` " - -cat /proc/$$/status | grep Cpus_allowed_list -cat /proc/meminfo - -echo "Array Allocation Number: $SLURM_ARRAY_JOB_ID" -echo "Array Index: $SLURM_ARRAY_TASK_ID" - - -module load anacondapy/5.3.1 -module load elastix/4.8 -. activate lightsheet - -#set up dictionary and save -OUT0=$(sbatch --array=0 slurm_files/step0.sh) -echo $OUT0 - -#run elastix, note this will not update any orientation changes, useful for new atlas -OUT3=$(sbatch --dependency=afterany:${OUT0##* } --array=0-2 slurm_files/step3.sh) -echo $OUT3 - - -# Usage notes: -# after = go once the specified job starts -# afterany = go if the specified job finishes, regardless of success -# afternotok = go if the specified job fails -# afterok = go if the specified job completes successfully diff --git a/test_registration.sh b/test_registration.sh deleted file mode 100644 index 40943ab7..00000000 --- a/test_registration.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/env bash -# -#SBATCH -p all # partition (queue) -#SBATCH -n 1 # number of cores -#SBATCH -t 20 # time (minutes) -#SBATCH -o logs/outmain_registration.out # STDOUT -#SBATCH -e logs/outmain_registration.err # STDERR - - -echo "In the directory: `pwd` " -echo "As the user: `whoami` " -echo "on host: `hostname` " - -cat /proc/$$/status | grep Cpus_allowed_list -cat /proc/meminfo - -echo "Array Allocation Number: $SLURM_ARRAY_JOB_ID" -echo "Array Index: $SLURM_ARRAY_TASK_ID" - - -module load anacondapy/5.3.1 -module load elastix/4.8 -. activate lightsheet - -#set up dictionary and save -OUT0=$(sbatch --array=0 slurm_files/step0.sh) -echo $OUT0 - -#process zplns, check that 1000 > zplns/slurmfactor -OUT1=$(sbatch --dependency=afterany:${OUT0##* } --array=0-26 slurm_files/step1.sh) -echo $OUT1 - -#combine stacks into single tifffiles -OUT2=$(sbatch --dependency=afterany:${OUT1##* } --array=0 slurm_files/step2.sh) -echo $OUT2 - -#run elastix -OUT3=$(sbatch --dependency=afterany:${OUT2##* } --array=0-2 slurm_files/step3.sh) -echo $OUT3 - - -# Usage notes: -# after = go once the specified job starts -# afterany = go if the specified job finishes, regardless of success -# afternotok = go if the specified job fails -# afterok = go if the specified job completes successfully diff --git a/tools/utils/directorydeterminer.py b/tools/utils/directorydeterminer.py index 30c9ed04..c382d6b3 100644 --- a/tools/utils/directorydeterminer.py +++ b/tools/utils/directorydeterminer.py @@ -19,8 +19,6 @@ def directorydeterminer(): """ if socket.gethostname() == "spock-login.pni.princeton.edu": systemdirectory= "/jukebox/" - if socket.gethostname() == "PNI-1867vwtq2": - systemdirectory = "/jukebox/" else: systemdirectory = "/jukebox/"