Merge branch 'release-1.14-beta'

Conflicts:
	pydpiper/pipeline.py

INSTALL

@@ -1,6 +1,6 @@
 To install pydpiper, both the current source and most stable release can be downloaded from github:
 
-https://github.com/mfriedel/pydpiper
+https://github.com/Mouse-Imaging-Centre/pydpiper
 
 Using the Downloads button, access the .tar.gz or .zip file.
 Unzip and untar and change into the source directory (e.g. cd pydpiper)
@@ -14,7 +14,9 @@ ConfigArgParse - https://pypi.python.org/pypi/ConfigArgParse
 Py.test - http://pytest.org/
 graphviz - http://www.graphviz.org/
 pygraphviz - http://pygraphviz.github.io/
+minc-toolkit - https://github.com/BIC-MNI/minc-toolkit
 minc-stuffs - https://github.com/Mouse-Imaging-Centre/minc-stuffs
+pyminc - https://github.com/Mouse-Imaging-Centre/pyminc
 
 In Ubuntu (tested on 12.04), run: 
 	apt-get install python-networkx

NEWS

@@ -1,3 +1,51 @@
+New in Version 1.14
+===================
+* the build model and registration chain applications now both create a
+  composite image of the input files and print a message to the user,
+  allowing them to check that the orientation is sufficiently similar
+  that the LSQ6 registration method chosen is likely to succeed
+* memory requirements for mincblur and nonlinear minctracc stages are now
+  estimated dynamically before such a stage runs.  The nlin minctracc protocol
+  now accepts a "memory" parameter allowing you to override the per-voxel
+  default (based on the native-resolution call) with a fixed amount in GB
+  per minctracc generation, e.g.: '"memory";0.5;1;1;1.5;2' dependending
+  on the particularities of your images or to get better estimates than those
+  based on the finest resolution
+* due to poor performance on SciNet, blur calls running on PBS queues now
+  use pmincaverage instead of mincaverage.  This adds a dependency on pyminc;
+  see the INSTALL file.
+* PydPiper's understanding of the "gradient" variable in the minctracc
+  nonlinear protocols and the internal `useGradient` variable have changed:
+  "FALSE"/False (resp.) causes the blurring to be applied to the image intensities;
+  "TRUE"/True causes it to be applied to the image gradients, vs. the previous
+  behaviour (True implied blurring of *both* the intensities and the gradient)
+* changes to the supplied minctracc nonlinear protocol, partly to reflect the
+  above gradient-related changes
+* the registration chain now allows you to specify --average-time-point=-1
+  to use the last set of scans for inter-subject registration
+* additional checking of input files
+* a test script which exercises the MBM and MAGeT applications is
+  now provided with instructions in pydpiper_testing/
+* SGE executors now request memory dynamically based on the requirements of
+  the runnable stages.  In this configuration, the --mem option now gives
+  a maximum amount allowed per executor.  The system exits if jobs require
+  more memory than this maximum.
+* on PBS, the remote server now writes its stdout to a text file in the working
+  directory, so the user needn't dig into, e.g., SciNet's /var to examine
+  it while the pipeline is still running
+* a few changes to defaults: default memory is now 1.75G/job rather than 2;
+  PBS executors wait for up to 3 minutes for the server to start instead of 50s;
+  a few changes to the supplied config files
+* better handling of csv files with extra commas in the registration chain
+* various bugfixes, in particular to the registration chain
+* some improved logic w.r.t. updating state of stages after failure of a
+  dependency
+* the internal queue has been changed to a set to support additional operations
+  related to scheduling and resource management (particularly the dynamic
+  executor memory requests; see above)
+* miscellaneous code improvements
+
+
 New in Version 1.13
 ===================
 * a composite image is now created after the LSQ6 stage and a message output,

applications/MAGeT.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
 from pydpiper.application import AbstractApplication
 import pydpiper.file_handling as fh
 from atoms_and_modules.registration_file_handling import RegistrationPipeFH
-from atoms_and_modules.registration_functions import initializeInputFiles, addGenRegArgumentGroup, checkThatInputFilesAreProvided
+from atoms_and_modules.registration_functions import initializeInputFiles, addGenRegArgumentGroup, checkInputFiles
 from atoms_and_modules.MAGeT_modules import MAGeTMask, MAGeTRegister, voxelVote, addMAGeTArgumentGroup
 from atoms_and_modules.LSQ12 import addLSQ12ArgumentGroup
 from atoms_and_modules.NLIN import addNlinRegArgumentGroup
@@ -34,21 +35,21 @@ def setup_appName(self):
         return appName
 
     def run(self):
-        checkThatInputFilesAreProvided(self.args)
+        checkInputFiles(self.args)
 
         if self.options.reg_method != "minctracc" and self.options.reg_method != "mincANTS":
-            print "Incorrect registration method specified: ", self.options.reg_method
+            print("Incorrect registration method specified: ", self.options.reg_method)
             sys.exit()
 
         # given that the lsq12 and nlin protocols are hard coded at the moment, exit if we are not at 
         # MICe, and provide some information as to where to find them
         if not exists(self.options.lsq12_protocol):
-            print "The lsq12 protocol does not exists: ", self.options.lsq12_protocol
-            print "You can find the default MAGeT protocols in your pydpiper source directory, in: applications_testing/test_data/"
+            print("The lsq12 protocol does not exists: ", self.options.lsq12_protocol)
+            print("You can find the default MAGeT protocols in your pydpiper source directory, in: applications_testing/test_data/")
             sys.exit()
         if not exists(self.options.nlin_protocol):
-            print "The nlin protocol does not exists: ", self.options.nlin_protocol
-            print "You can find the default MAGeT protocols in your pydpiper source directory, in: applications_testing/test_data/"
+            print("The nlin protocol does not exists: ", self.options.nlin_protocol)
+            print("You can find the default MAGeT protocols in your pydpiper source directory, in: applications_testing/test_data/")
             sys.exit()
 
         # There are two variables that can be used to determine where the output
@@ -80,11 +81,11 @@ def run(self):
                 average.append(abspath(inFile))
         # check to make sure len(average)==len(labels)
         if (not len(average) == len(labels)) or (not len(average) == len(masks)):
-            print "\nError: not all atlases/labels/masks match."
-            print "The allowed naming conventions are:\n{atlas}.mnc\n{atlas}_labels.mnc\n{atlas}_mask.mnc"
-            print "\nand:\n{atlas}_average.mnc\n{atlas}_labels.mnc\n{atlas}_mask.mnc\n"
-            print "Check the atlas library directory: " + str(self.options.atlas_lib) + " and try again."
-            print "Exiting..."
+            print("\nError: not all atlases/labels/masks match.")
+            print("The allowed naming conventions are:\n{atlas}.mnc\n{atlas}_labels.mnc\n{atlas}_mask.mnc")
+            print("\nand:\n{atlas}_average.mnc\n{atlas}_labels.mnc\n{atlas}_mask.mnc\n")
+            print("Check the atlas library directory: " + str(self.options.atlas_lib) + " and try again.")
+            print("Exiting...")
             sys.exit() 
         else:
         # match labels with averages
@@ -107,8 +108,8 @@ def run(self):
 
         # exit if no atlases were found in the specified directory
         if numLibraryAtlases == 0:
-            print "\nError: no atlases were found in the specified directory: %s" % self.options.atlas_lib
-            print "Exiting..."
+            print("\nError: no atlases were found in the specified directory: %s" % self.options.atlas_lib)
+            print("Exiting...")
             sys.exit()
 
         #MF TODO: add some checking to make sure that atlas/labels/naming all worked correctly

applications/MBM.py

@@ -7,19 +7,17 @@
 import atoms_and_modules.LSQ6 as lsq6
 import atoms_and_modules.LSQ12 as lsq12
 import atoms_and_modules.NLIN as nlin
-import atoms_and_modules.minc_parameters as mp
 import atoms_and_modules.stats_tools as st
 from atoms_and_modules.minc_modules import createQualityControlImages
 import os
 import logging
 from datetime import date
-import time
 
 
 logger = logging.getLogger(__name__)
 
 def addMBMGroup(parser):
-    group = parser.add_argument_group("MBM options", "Options for MICe-build-model.")
+    parser.add_argument_group("MBM options", "Options for MICe-build-model.")
 
 class MBMApplication(AbstractApplication):
     def setup_options(self):
@@ -39,7 +37,7 @@ def run(self):
         options = self.options
         args = self.args
 
-        rf.checkThatInputFilesAreProvided(args)
+        rf.checkInputFiles(args)
 
         # make sure only one of the lsq6 target options is provided
         lsq6.verifyCorrectLSQ6TargetOptions(options.bootstrap,
@@ -72,13 +70,29 @@ def run(self):
                                                       target_file_directory,
                                                       self.outputDir,
                                                       options.pipeline_name)
-
+
+        # We will create an initial verification image here. This
+        # will show the user whether it is likely for the alignment to
+        # work altogether (potentially the orientation of the target
+        # and the input files is too different.) 
+        montageBeforeRegistration = self.outputDir + "/" + options.pipeline_name + "_quality_control_target_and_inputfiles.png"
+        initialVerificationImages = createQualityControlImages([targetPipeFH] + inputFiles,
+                                                               montageOutPut=montageBeforeRegistration,
+                                                               message="at the very start of the registration.")
+        self.pipeline.addPipeline(initialVerificationImages.p)
+
         #LSQ6 MODULE, NUC and INORM
         runLSQ6NucInorm = lsq6.LSQ6NUCInorm(inputFiles,
                                             targetPipeFH,
                                             initModel, 
                                             dirs.lsq6Dir, 
                                             options)
+        # TODO: This is a temporary hack: we add the output of the verification 
+        # image as an input to all stages from the LSQ6 stage to make
+        # sure that this image is created as soon as possible
+        # This obviously is overkill, but it doens't really hurt either
+        for lsq6Stage in runLSQ6NucInorm.p.stages:
+            lsq6Stage.inputFiles.append(montageBeforeRegistration)
         self.pipeline.addPipeline(runLSQ6NucInorm.p)
 
         # At this point in the pipeline it's important to check the 
@@ -88,9 +102,15 @@ def run(self):
         # a verification image showing slices through all files
         montageLSQ6 = self.outputDir + "/" + options.pipeline_name + "_quality_control_montage_lsq6.png"
         # TODO, base scaling factor on resolution of initial model or target
-        lsq6VerificationImages = createQualityControlImages(inputFiles,
+        # add the LSQ6 average file as well:
+        filesToCreateImagesFrom = []
+        if runLSQ6NucInorm.lsq6Avg:
+            filesToCreateImagesFrom = [runLSQ6NucInorm.lsq6Avg] + inputFiles
+        else:
+            filesToCreateImagesFrom = inputFiles
+        lsq6VerificationImages = createQualityControlImages(filesToCreateImagesFrom,
                                                             montageOutPut=montageLSQ6,
-                                                            stage="lsq6")
+                                                            message="after the lsq6 alignment")
         self.pipeline.addPipeline(lsq6VerificationImages.p)
 
         # LSQ12 MODULE
@@ -113,13 +133,20 @@ def run(self):
 
 
         lsq12module = lsq12.FullLSQ12(inputFiles, 
-                                      dirs.lsq12Dir, 
+                                      dirs.lsq12Dir,
+                                      queue_type=options.queue_type,
                                       likeFile=targetPipeFH, 
                                       maxPairs=options.lsq12_max_pairs, 
                                       lsq12_protocol=options.lsq12_protocol,
                                       subject_matter=options.lsq12_subject_matter,
                                       resolution=resolutionForLSQ12)
         lsq12module.iterate()
+        # TODO: This is also a temporary hack: we add the output of the verification 
+        # image as an input to all stages from the LSQ12 stage to make
+        # sure that this image is created as soon as possible
+        # This obviously is overkill, but it doens't really hurt either
+        for lsq12Stage in lsq12module.p.stages:
+            lsq12Stage.inputFiles.append(montageLSQ6)
         self.pipeline.addPipeline(lsq12module.p)
 
         #TODO: Additional NUC step here. This will impact both the lsq6 and lsq12 modules. 

applications/pairwise_nlin.py

@@ -7,7 +7,6 @@
 import atoms_and_modules.minc_modules as mm
 import atoms_and_modules.minc_atoms as ma
 import atoms_and_modules.stats_tools as st
-import atoms_and_modules.option_groups as og
 from datetime import date
 from os.path import abspath, isdir
 import logging
@@ -28,7 +27,6 @@ def setup_options(self):
         self.parser.add_argument_group(group)
         """Add option groups from specific modules"""
         rf.addGenRegArgumentGroup(self.parser)
-        og.tmpLongitudinalArgumentGroup(self.parser)
         st.addStatsArguments(self.parser)
 
     def setup_appName(self):

applications/pipeline.log

@@ -0,0 +1 @@
+[2015-03-20 14:35:59.510,Pyro4,INFO] Pyro log configured using built-in defaults, level=INFO