Combine refactor

pypeit/calibrations.py

@@ -4,6 +4,7 @@
 .. include common links, assuming primary doc root is up one directory
 .. include:: ../include/links.rst
 """
+import os
 from pathlib import Path
 from datetime import datetime
 from copy import deepcopy
@@ -209,6 +210,46 @@ def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None,
         self.success = False
         self.failed_step = None
 
+    def check_calibrations(self, file_list, check_lamps=True):
+        """
+        Check if the input calibration files are consistent with each other.
+        This step is usually needed when combining calibration frames of a given type.
+        This routine currently only prints out warning messages if the calibration files are not consistent.
+
+        Note: The exposure times are currently checked in the combine step, so they are not checked here.
+
+        Parameters
+        ----------
+        file_list : list
+            List of calibration files to check
+        check_lamps : bool, optional
+            Check if the lamp status is the same for all the files. Default is True.
+        """
+
+        lampstat = [None] * len(file_list)
+        # Loop on the files
+        for ii, ifile in enumerate(file_list):
+            # Save the lamp status
+            headarr = deepcopy(self.spectrograph.get_headarr(ifile))
+            lampstat[ii] = self.spectrograph.get_lamps_status(headarr)
+
+        # Check that the lamps being combined are all the same
+        if check_lamps:
+            if not lampstat[1:] == lampstat[:-1]:
+                msgs.warn("The following files contain different lamp status")
+                # Get the longest strings
+                maxlen = max([len("Filename")] + [len(os.path.split(x)[1]) for x in file_list])
+                maxlmp = max([len("Lamp status")] + [len(x) for x in lampstat])
+                strout = "{0:" + str(maxlen) + "}  {1:s}"
+                # Print the messages
+                print(msgs.indent() + '-' * maxlen + "  " + '-' * maxlmp)
+                print(msgs.indent() + strout.format("Filename", "Lamp status"))
+                print(msgs.indent() + '-' * maxlen + "  " + '-' * maxlmp)
+                for ff, file in enumerate(file_list):
+                    print(msgs.indent()
+                          + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_"))))
+                print(msgs.indent() + '-' * maxlen + "  " + '-' * maxlmp)
+
     def find_calibrations(self, frametype, frameclass):
         """
         Find calibration files and identifiers.
@@ -334,6 +375,9 @@ def get_arc(self):
         # Reset the BPM
         self.get_bpm(frame=raw_files[0])
 
+        # Perform a check on the files
+        self.check_calibrations(raw_files)
+
         # Otherwise, create the processed file.
         msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
         self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det,
@@ -377,6 +421,9 @@ def get_tiltimg(self):
         # Reset the BPM
         self.get_bpm(frame=raw_files[0])
 
+        # Perform a check on the files
+        self.check_calibrations(raw_files)
+
         # Otherwise, create the processed file.
         msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
         self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det,
@@ -426,6 +473,9 @@ def get_align(self):
         # Reset the BPM
         self.get_bpm(frame=raw_files[0])
 
+        # Perform a check on the files
+        self.check_calibrations(raw_files)
+
         # Otherwise, create the processed file.
         msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
         msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det,
@@ -473,6 +523,9 @@ def get_bias(self):
             self.msbias = frame['class'].from_file(cal_file, chk_version=self.chk_version)
             return self.msbias
 
+        # Perform a check on the files
+        self.check_calibrations(raw_files)
+
         # Otherwise, create the processed file.
         msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
         self.msbias = buildimage.buildimage_fromlist(self.spectrograph, self.det,
@@ -523,6 +576,9 @@ def get_dark(self):
         # there any reason why creation of the bpm should come after the dark,
         # or can we change the order?
 
+        # Perform a check on the files
+        self.check_calibrations(raw_files)
+
         # Otherwise, create the processed file.
         self.msdark = buildimage.buildimage_fromlist(self.spectrograph, self.det,
                                                      self.par['darkframe'], raw_files,
@@ -597,6 +653,9 @@ def get_scattlight(self):
         # Reset the BPM
         self.get_bpm(frame=raw_scattlight_files[0])
 
+        # Perform a check on the files
+        self.check_calibrations(raw_scattlight_files)
+
         binning = self.fitstbl[scatt_idx[0]]['binning']
         dispname = self.fitstbl[scatt_idx[0]]['dispname']
         scattlightImage = buildimage.buildimage_fromlist(self.spectrograph, self.det,
@@ -725,6 +784,10 @@ def get_flats(self):
         if len(raw_pixel_files) > 0:
             # Reset the BPM
             self.get_bpm(frame=raw_pixel_files[0])
+
+            # Perform a check on the files
+            self.check_calibrations(raw_pixel_files)
+
             msgs.info('Creating pixel-flat calibration frame using files: ')
             for f in raw_pixel_files:
                 msgs.prindent(f'{Path(f).name}')
@@ -737,6 +800,10 @@ def get_flats(self):
             if len(raw_lampoff_files) > 0:
                 # Reset the BPM
                 self.get_bpm(frame=raw_lampoff_files[0])
+
+                # Perform a check on the files
+                self.check_calibrations(raw_lampoff_files)
+
                 msgs.info('Subtracting lamp off flats using files: ')
                 for f in raw_lampoff_files:
                     msgs.prindent(f'{Path(f).name}')
@@ -763,6 +830,10 @@ def get_flats(self):
         if not pix_is_illum and len(raw_illum_files) > 0:
             # Reset the BPM
             self.get_bpm(frame=raw_illum_files[0])
+
+            # Perform a check on the files
+            self.check_calibrations(raw_illum_files)
+
             msgs.info('Creating slit-illumination flat calibration frame using files: ')
             for f in raw_illum_files:
                 msgs.prindent(f'{Path(f).name}')
@@ -775,6 +846,10 @@ def get_flats(self):
                 for f in raw_lampoff_files:
                     msgs.prindent(f'{Path(f).name}')
                 if lampoff_flat is None:
+                    # Perform a check on the files
+                    self.check_calibrations(raw_lampoff_files)
+
+                    # Build the image
                     lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det,
                                                                   self.par['lampoffflatsframe'],
                                                                   raw_lampoff_files,
@@ -889,6 +964,9 @@ def get_slits(self):
         # Reset the BPM
         self.get_bpm(frame=raw_trace_files[0])
 
+        # Perform a check on the files
+        self.check_calibrations(raw_trace_files)
+
         traceImage = buildimage.buildimage_fromlist(self.spectrograph, self.det,
                                                     self.par['traceframe'], raw_trace_files,
                                                     bias=self.msbias, bpm=self.msbpm,
@@ -903,6 +981,9 @@ def get_slits(self):
             # Reset the BPM
             self.get_bpm(frame=raw_trace_files[0])
 
+            # Perform a check on the files
+            self.check_calibrations(raw_lampoff_files)
+
             lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det,
                                                           self.par['lampoffflatsframe'],
                                                           raw_lampoff_files, dark=self.msdark,

pypeit/images/buildimage.py

@@ -10,11 +10,13 @@
 
 from pypeit import msgs
 from pypeit.par import pypeitpar
+from pypeit.images import rawimage
 from pypeit.images import combineimage
 from pypeit.images import pypeitimage
 from pypeit.core.framematch import valid_frametype
 
 
+
 class ArcImage(pypeitimage.PypeItCalibrationImage):
     """
     Simple DataContainer for the Arc Image
@@ -160,7 +162,10 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
                         scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True,
                         slits=None, mosaic=None, calib_dir=None, setup=None, calib_id=None):
     """
-    Perform basic image processing on a list of images and combine the results.
+    Perform basic image processing on a list of images and combine the results. All 
+    core processing steps for each image are handled by :class:`~pypeit.images.rawimage.RawImage` and
+    image combination is handled by :class:`~pypeit.images.combineimage.CombineImage`.
+    This function can be used to process both single images, lists of images, and detector mosaics.
 
     .. warning::
 
@@ -246,16 +251,20 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
     # Should the detectors be reformatted into a single image mosaic?
     if mosaic is None:
         mosaic = isinstance(det, tuple) and frame_par['frametype'] not in ['bias', 'dark']
+
+    rawImage_list = []
+    # Loop on the files
+    for ifile in file_list:
+        # Load raw image
+        rawImage = rawimage.RawImage(ifile, spectrograph, det)
+        # Process
+        rawImage_list.append(rawImage.process(
+            frame_par['process'], scattlight=scattlight, bias=bias, 
+            bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic))
 
     # Do it
-    combineImage = combineimage.CombineImage(spectrograph, det, frame_par['process'], file_list)
-    pypeitImage = combineImage.run(bias=bias, bpm=bpm, dark=dark, flatimages=flatimages,
-                                   scattlight=scattlight, sigma_clip=frame_par['process']['clip'],
-                                   sigrej=frame_par['process']['comb_sigrej'],
-                                   maxiters=maxiters, ignore_saturation=ignore_saturation,
-                                   slits=slits, combine_method=frame_par['process']['combine'],
-                                   mosaic=mosaic)
-
+    combineImage = combineimage.CombineImage(rawImage_list, frame_par['process'])
+    pypeitImage = combineImage.run(maxiters=maxiters, ignore_saturation=ignore_saturation)
     # Return class type, if returning any of the frame_image_classes
     cls = frame_image_classes[frame_par['frametype']] \
             if frame_par['frametype'] in frame_image_classes.keys() else None