Combine refactor

pypeit/images/buildimage.py

@@ -10,11 +10,13 @@
 
 from pypeit import msgs
 from pypeit.par import pypeitpar
+from pypeit.images import rawimage
 from pypeit.images import combineimage
 from pypeit.images import pypeitimage
 from pypeit.core.framematch import valid_frametype
 
 
+
 class ArcImage(pypeitimage.PypeItCalibrationImage):
     """
     Simple DataContainer for the Arc Image
@@ -160,7 +162,10 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
                         scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True, slits=None,
                         mosaic=None, calib_dir=None, setup=None, calib_id=None):
     """
-    Perform basic image processing on a list of images and combine the results.
+    Perform basic image processing on a list of images and combine the results. All 
+    core processing steps for each image are handled by :class:`~pypeit.images.rawimage.RawImage` and
+    image combination is handled by :class:`~pypeit.images.combineimage.CombineImage`.
+    This function can be used to process both single images, lists of images, and detector mosaics.
 
     .. warning::
 
@@ -246,16 +251,20 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
     # Should the detectors be reformatted into a single image mosaic?
     if mosaic is None:
         mosaic = isinstance(det, tuple) and frame_par['frametype'] not in ['bias', 'dark']
-
+
+    rawImage_list = []
+    # Loop on the files
+    for ifile in file_list:
+        # Load raw image
+        rawImage = rawimage.RawImage(ifile, spectrograph, det)
+        # Process
+        rawImage_list.append(rawImage.process(
+            frame_par['process'], scattlight=scattlight, bias=bias, 
+            bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic))
+
     # Do it
-    combineImage = combineimage.CombineImage(spectrograph, det, frame_par['process'], file_list)
-    pypeitImage = combineImage.run(bias=bias, bpm=bpm, dark=dark, flatimages=flatimages, scattlight=scattlight,
-                                   sigma_clip=frame_par['process']['clip'],
-                                   sigrej=frame_par['process']['comb_sigrej'],
-                                   maxiters=maxiters, ignore_saturation=ignore_saturation,
-                                   slits=slits, combine_method=frame_par['process']['combine'],
-                                   mosaic=mosaic)
-
+    combineImage = combineimage.CombineImage(rawImage_list, spectrograph, frame_par['process'])
+    pypeitImage = combineImage.run(maxiters=maxiters, ignore_saturation=ignore_saturation)
     # Return class type, if returning any of the frame_image_classes
     cls = frame_image_classes[frame_par['frametype']] \
             if frame_par['frametype'] in frame_image_classes.keys() else None

pypeit/images/combineimage.py

@@ -16,28 +16,21 @@
 from pypeit.par import pypeitpar
 from pypeit import utils
 from pypeit.images import pypeitimage
-from pypeit.images import rawimage
 from pypeit.images import imagebitmask
 
 class CombineImage:
     """
-    Process and combine detector images.
-
-    All core processing steps for each image are handled by
-    :class:`~pypeit.images.rawimage.RawImage`.  This class can be used to
-    process both single images, lists of images, and detector mosaics.
+    Process and combine detector images. 
 
     Args:
-        spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`):
-            Spectrograph used to take the data.
-        det (:obj:`int`, :obj:`tuple`):
-            The 1-indexed detector number(s) to process.  If a tuple, it must
-            include detectors viable as a mosaic for the provided spectrograph;
-            see :func:`~pypeit.spectrographs.spectrograph.Spectrograph.allowed_mosaics`.
+            Parameters that dictate the processing of the images.
+        rawImages (:obj:`list`):
+            Either a single :class:`~pypeit.images.rawimage.RawImage` object or a list of one or more 
+            of these objects to be combined into a an image.
+       spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`):
+            Spectrograph used to take the data.            
         par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`):
             Parameters that dictate the processing of the images.
-        files (:obj:`str`, array-like):
-            A set of one or more images to process and combine.
 
     Attributes:
         spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`):
@@ -46,24 +39,25 @@ class CombineImage:
             The 1-indexed detector number(s) to process.
         par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`):
             Parameters that dictate the processing of the images.
-        files (:obj:`list`):
-            A set of one or more images to process and combine.
+        rawImages (:obj:`list`):
+            A list of one or more :class:`~pypeit.images.rawimage.RawImage` objects 
+            to be combined.             
 
     """
-    def __init__(self, spectrograph, det, par, files):
-        self.spectrograph = spectrograph
-        self.det = det
+    def __init__(self, rawImages, spectrograph, par):
         if not isinstance(par, pypeitpar.ProcessImagesPar):
             msgs.error('Provided ParSet for must be type ProcessImagesPar.')
+        self.rawImages = list(rawImages) if hasattr(rawImages, '__len__') else [rawImages]
+        self.spectrograph = spectrograph
         self.par = par  # This musts be named this way as it is frequently a child
-        self.files = list(files) if hasattr(files, '__len__') else [files]
-        # NOTE: nfiles is a property method.  Defining files above must come
+
+        # NOTE: nimgs is a property method.  Defining rawImages above must come
         # before this check!
-        if self.nfiles == 0:
+        if self.nimgs == 0:
             msgs.error('CombineImage requires a list of files to instantiate')
 
-    def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=False, sigma_clip=True,
-            bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='mean', mosaic=False):
+
+    def run(self, ignore_saturation=False, maxiters=5):
         r"""
         Process and combine all images.
 
@@ -75,7 +69,7 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
         file and returns the result.
 
         If there are multiple files, all the files are processed and the
-        processed images are combined based on the ``combine_method``, where the
+        processed images are combined based on the ``par['combine']``, where the
         options are:
 
             - 'mean': If ``sigma_clip`` is True, this is a sigma-clipped mean;
@@ -133,112 +127,77 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
             in images of the same shape.
 
         Args:
-            bias (:class:`~pypeit.images.buildimage.BiasImage`, optional):
-                Bias image for bias subtraction; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            scattlight (:class:`~pypeit.scattlight.ScatteredLight`, optional):
-                Scattered light model to be used to determine scattered light.
-            flatimages (:class:`~pypeit.flatfield.FlatImages`, optional):
-                Flat-field images for flat fielding; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
             ignore_saturation (:obj:`bool`, optional):
                 If True, turn off the saturation flag in the individual images
                 before stacking.  This avoids having such values set to 0, which
                 for certain images (e.g. flat calibrations) can have unintended
                 consequences.
-            sigma_clip (:obj:`bool`, optional):
-                When ``combine_method='mean'``, perform a sigma-clip the data;
-                see :func:`~pypeit.core.combine.weighted_combine`.
-            bpm (`numpy.ndarray`_, optional):
-                Bad pixel mask; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            sigrej (:obj:`float`, optional):
-                When ``combine_method='mean'``, this sets the sigma-rejection
-                thresholds used when sigma-clipping the image combination.
-                Ignored if ``sigma_clip`` is False.  If None and ``sigma_clip``
-                is True, the thresholds are determined automatically based on
-                the number of images provided; see
-                :func:`~pypeit.core.combine.weighted_combine``.
             maxiters (:obj:`int`, optional):
-                When ``combine_method='mean'``) and sigma-clipping
+                When ``par['combine']='mean'``) and sigma-clipping
                 (``sigma_clip`` is True), this sets the maximum number of
                 rejection iterations.  If None, rejection iterations continue
                 until no more data are rejected; see
                 :func:`~pypeit.core.combine.weighted_combine``.
-            slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional):
-                Slit edge trace locations; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            dark (:class:`~pypeit.images.buildimage.DarkImage`, optional):
-                Dark-current image; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            combine_method (:obj:`str`, optional):
-                Method used to combine images.  Must be ``'mean'`` or
-                ``'median'``; see above.
-            mosaic (:obj:`bool`, optional):
-                If multiple detectors are being processes, mosaic them into a
-                single image.  See
-                :func:`~pypeit.images.rawimage.RawImage.process`.
 
         Returns:
             :class:`~pypeit.images.pypeitimage.PypeItImage`: The combination of
             all the processed images.
         """
+
+
         # Check the input (i.e., bomb out *before* it does any processing)
-        if self.nfiles == 0:
+        if self.nimgs == 0:
             msgs.error('Object contains no files to process!')
-        if self.nfiles > 1 and combine_method not in ['mean', 'median']:
-            msgs.error(f'Unknown image combination method, {combine_method}.  Must be '
+        if self.nimgs > 1 and self.par['combine'] not in ['mean', 'median']:
+            msgs.error(f'Unknown image combination method, {self.par["combine"]}.  Must be '
                        '"mean" or "median".')
-
+        file_list = []
         # Loop on the files
-        for kk, ifile in enumerate(self.files):
-            # Load raw image
-            rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det)
-            # Process
-            pypeitImage = rawImage.process(self.par, scattlight=scattlight, bias=bias, bpm=bpm, dark=dark,
-                                           flatimages=flatimages, slits=slits, mosaic=mosaic)
-
-            if self.nfiles == 1:
+        for kk, rawImage in enumerate(self.rawImages):
+            if self.nimgs == 1:
                 # Only 1 file, so we're done
-                pypeitImage.files = self.files
-                return pypeitImage
+                rawImage.files = [rawImage.filename]
+                return rawImage
             elif kk == 0:
                 # Allocate arrays to collect data for each frame
-                shape = (self.nfiles,) + pypeitImage.shape
+                shape = (self.nimgs,) + rawImage.shape
                 img_stack = np.zeros(shape, dtype=float)
                 scl_stack = np.ones(shape, dtype=float)
                 rn2img_stack = np.zeros(shape, dtype=float)
                 basev_stack = np.zeros(shape, dtype=float)
                 gpm_stack = np.zeros(shape, dtype=bool)
-                lampstat = [None]*self.nfiles
-                exptime = np.zeros(self.nfiles, dtype=float)
+                lampstat = [None]*self.nimgs
+                exptime = np.zeros(self.nimgs, dtype=float)
 
             # Save the lamp status
             # TODO: As far as I can tell, this is the *only* place rawheadlist
             # is used.  Is there a way we can get this from fitstbl instead?
-            lampstat[kk] = self.spectrograph.get_lamps_status(pypeitImage.rawheadlist)
+            lampstat[kk] = self.spectrograph.get_lamps_status(rawImage.rawheadlist)
             # Save the exposure time to check if it's consistent for all images.
-            exptime[kk] = pypeitImage.exptime
+            exptime[kk] = rawImage.exptime
             # Processed image
-            img_stack[kk] = pypeitImage.image
+            img_stack[kk] = rawImage.image
             # Get the count scaling
-            if pypeitImage.img_scale is not None:
-                scl_stack[kk] = pypeitImage.img_scale
+            if rawImage.img_scale is not None:
+                scl_stack[kk] = rawImage.img_scale
             # Read noise squared image
-            if pypeitImage.rn2img is not None:
-                rn2img_stack[kk] = pypeitImage.rn2img * scl_stack[kk]**2
+            if rawImage.rn2img is not None:
+                rn2img_stack[kk] = rawImage.rn2img * scl_stack[kk]**2
             # Processing variance image
-            if pypeitImage.base_var is not None:
-                basev_stack[kk] = pypeitImage.base_var * scl_stack[kk]**2
+            if rawImage.base_var is not None:
+                basev_stack[kk] = rawImage.base_var * scl_stack[kk]**2
             # Final mask for this image
             # TODO: This seems kludgy to me. Why not just pass ignore_saturation
             # to process_one and ignore the saturation when the mask is actually
             # built, rather than untoggling the bit here?
             if ignore_saturation:  # Important for calibrations as we don't want replacement by 0
-                pypeitImage.update_mask('SATURATION', action='turn_off')
+                rawImage.update_mask('SATURATION', action='turn_off')
             # Get a simple boolean good-pixel mask for all the unmasked pixels
-            gpm_stack[kk] = pypeitImage.select_flag(invert=True)
+            gpm_stack[kk] = rawImage.select_flag(invert=True)
+            file_list.append(rawImage.filename)
 
+        # TODO JFH: This should not be here, but rather should be done in the Calibrations class. Putting in calibration specific
+        # lamp checking in an image combining class is not ideal. 
         # Check that the lamps being combined are all the same:
         if not lampstat[1:] == lampstat[:-1]:
             msgs.warn("The following files contain different lamp status")
@@ -265,21 +224,21 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
             comb_texp = exptime[0]
 
         # Coadd them
-        if combine_method == 'mean':
-            weights = np.ones(self.nfiles, dtype=float)/self.nfiles
+        if self.par['combine'] == 'mean':
+            weights = np.ones(self.nimgs, dtype=float)/self.nimgs
             img_list_out, var_list_out, gpm, nframes \
                     = combine.weighted_combine(weights,
                                                [img_stack, scl_stack],  # images to stack
                                                [rn2img_stack, basev_stack], # variances to stack
-                                               gpm_stack, sigma_clip=sigma_clip,
+                                               gpm_stack, sigma_clip=self.par['clip'],
                                                sigma_clip_stack=img_stack,  # clipping based on img
-                                               sigrej=sigrej, maxiters=maxiters)
+                                               sigrej=self.par['comb_sigrej'], maxiters=maxiters)
             comb_img, comb_scl = img_list_out
             comb_rn2, comb_basev = var_list_out
             # Divide by the number of images that contributed to each pixel
             comb_scl[gpm] /= nframes[gpm]
 
-        elif combine_method == 'median':
+        elif self.par['combine'] == 'median':
             bpm_stack = np.logical_not(gpm_stack)
             nframes = np.sum(gpm_stack, axis=0)
             gpm = nframes > 0
@@ -310,26 +269,26 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
 
         # Build the combined image
         comb = pypeitimage.PypeItImage(image=comb_img, ivar=utils.inverse(comb_var), nimg=nframes,
-                                       amp_img=pypeitImage.amp_img, det_img=pypeitImage.det_img,
+                                       amp_img=rawImage.amp_img, det_img=rawImage.det_img,
                                        rn2img=comb_rn2, base_var=comb_basev, img_scale=comb_scl,
                                        # NOTE: This *must* be a boolean.
                                        bpm=np.logical_not(gpm), 
                                        # NOTE: The detector is needed here so
                                        # that we can get the dark current later.
-                                       detector=pypeitImage.detector,
+                                       detector=rawImage.detector,
                                        PYP_SPEC=self.spectrograph.name,
                                        units='e-' if self.par['apply_gain'] else 'ADU',
                                        exptime=comb_texp, noise_floor=self.par['noise_floor'],
                                        shot_noise=self.par['shot_noise'])
 
         # Internals
         # TODO: Do we need these?
-        comb.files = self.files
-        comb.rawheadlist = pypeitImage.rawheadlist
-        comb.process_steps = pypeitImage.process_steps
+        comb.files = file_list
+        comb.rawheadlist = rawImage.rawheadlist
+        comb.process_steps = rawImage.process_steps
 
         # Build the base level mask
-        comb.build_mask(saturation='default', mincounts='default')
+        comb.build_mask(saturation='default' if not ignore_saturation else None, mincounts='default')
 
         # Flag all pixels with no contributions from any of the stacked images.
         comb.update_mask('STCKMASK', indx=np.logical_not(gpm))
@@ -338,10 +297,10 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
         return comb
 
     @property
-    def nfiles(self):
+    def nimgs(self):
         """
         The number of files in :attr:`files`.
         """
-        return len(self.files) if isinstance(self.files, (np.ndarray, list)) else 0
+        return len(self.rawImages) if isinstance(self.rawImages, (np.ndarray, list)) else 0
 
 

pypeit/images/pypeitimage.py

@@ -127,7 +127,8 @@ class PypeItImage(datamodel.DataContainer):
                  'shot_noise': dict(otype=bool, descr='Shot-noise included in variance'),
                  'spat_flexure': dict(otype=float,
                                       descr='Shift, in spatial pixels, between this image '
-                                            'and SlitTrace')}
+                                            'and SlitTrace'), 
+                 'filename': dict(otype=str, descr='Filename for the image'),}
     """Data model components."""
 
     internals = ['process_steps', 'files', 'rawheadlist']
@@ -162,8 +163,8 @@ def from_pypeitimage(cls, pypeitImage):
 
     def __init__(self, image, ivar=None, nimg=None, amp_img=None, det_img=None, rn2img=None,
                  base_var=None, img_scale=None, fullmask=None, detector=None, spat_flexure=None,
-                 PYP_SPEC=None, units=None, exptime=None, noise_floor=None, shot_noise=None,
-                 bpm=None, crmask=None, usermask=None, clean_mask=False):
+                 filename=None, PYP_SPEC=None, units=None, exptime=None, noise_floor=None, 
+                 shot_noise=None, bpm=None, crmask=None, usermask=None, clean_mask=False):
 
         if image is None:
             msgs.error('Must provide an image when instantiating PypeItImage.')