pre-merge fixes

doc/releases/1.16.1dev.rst

@@ -39,6 +39,11 @@ Script Changes
   expansion of and changes to the cache system.
 - Added ``pypeit_clean_cache`` script to facilitate both viewing and removing
   files in the cache.
+- A new script, called `pypeit_extract_datacube`, allows 1D spectra of point
+  sources to be extracted from datacubes.
+- The sensitivity function is now generated outside of datacube generation.
+- The `grating_corr` column is now used to select the correct grating
+  correction file for each spec2d file when generating the datacube.
 - Added ``pypeit_show_pixflat`` script to inspect the (slitless) pixel flat
   generated during the reduction and stored in ``data/static_calibs/{spectrograph_name}``
 
@@ -55,7 +60,10 @@ Under-the-hood Improvements
 Bug Fixes
 ---------
 
+- Fix "The system cannot find the file specified" errors when installing on Windows.
+- Fixed a fault caused when all frames in a pypeit file are identified as being
+  part of ``all`` calibration groups.
+- Allow for empty 2D wavecal solution in HDU extension of WaveCalib file
 - Fix a MAJOR BUT SUBTLE bug in the use of ``numpy.argsort``. When using ``numpy.argsort``
   the parameter kind='stable' should be used to ensure that a sorting algorithm more robust
-  than "quicksort" is used.
-
+  than "quicksort" is used.

pypeit/flatfield.py

@@ -1439,7 +1439,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s
         onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat
         # Setup
         slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1  # Need to +1 and -1 so that slitimg=-1 when off the slit
-        left, right, msk = self.slits.select_edges(initial=True, flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0)
+        left, right, msk = self.slits.select_edges(flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0)
         this_left = left[:, slit_idx]
         this_right = right[:, slit_idx]
         slitlen = int(np.median(this_right - this_left))
@@ -1607,7 +1607,72 @@ def spectral_illumination(self, gpm=None, debug=False):
                                       slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'],
                                       model=None, gpmask=gpm, skymask=None, trim=trim,
                                       flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0,
-                                      smooth_npix=self.flatpar['slit_illum_smooth_npix'])
+                                      smooth_npix=self.flatpar['slit_illum_smooth_npix'],
+                                      debug=debug)
+
+    def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', thresh=0.93, maxfrac=0.1, debug=False):
+        """
+        Tweak the slit edges based on the normalized slit illumination profile.
+
+        Args:
+            left (`numpy.ndarray`_):
+                Array with the left slit edge for a single slit. Shape is
+                :math:`(N_{\rm spec},)`.
+            right (`numpy.ndarray`_):
+                Array with the right slit edge for a single slit. Shape
+                is :math:`(N_{\rm spec},)`.
+            spat_coo (`numpy.ndarray`_):
+                Spatial pixel coordinates in fractions of the slit width
+                at each spectral row for the provided normalized flat
+                data. Coordinates are relative to the left edge (with the
+                left edge at 0.). Shape is :math:`(N_{\rm flat},)`.
+                Function assumes the coordinate array is sorted.
+            norm_flat (`numpy.ndarray`_)
+                Normalized flat data that provide the slit illumination
+                profile. Shape is :math:`(N_{\rm flat},)`.
+            method (:obj:`str`, optional):
+                Method to use for tweaking the slit edges. Options are:
+                    - 'threshold': Use the threshold to set the slit edge
+                        and then shift it to the left or right based on the
+                        illumination profile.
+                    - 'gradient': Use the gradient of the illumination
+                        profile to set the slit edge and then shift it to
+                        the left or right based on the illumination profile.
+            thresh (:obj:`float`, optional):
+                Threshold of the normalized flat profile at which to
+                place the two slit edges.
+            maxfrac (:obj:`float`, optional):
+                The maximum fraction of the slit width that the slit edge
+                can be adjusted by this algorithm. If ``maxfrac = 0.1``,
+                this means the maximum change in the slit width (either
+                narrowing or broadening) is 20% (i.e., 10% for either
+                edge).
+            debug (:obj:`bool`, optional):
+                Show flow interrupting plots that show illumination
+                profile in the case of a failure and the placement of the
+                tweaked edge for each side of the slit regardless.
+
+        Returns:
+            tuple: Returns six objects:
+
+                - The threshold used to set the left edge
+                - The fraction of the slit that the left edge is shifted to
+                  the right
+                - The adjusted left edge
+                - The threshold used to set the right edge
+                - The fraction of the slit that the right edge is shifted to
+                  the left
+                - The adjusted right edge
+        """
+        # TODO :: Since this is just a wrapper, and not really "core", maybe it should be moved to pypeit.flatfield?
+        # Tweak the edges via the specified method
+        if method == "threshold":
+            return flat.tweak_slit_edges_threshold(left, right, spat_coo, norm_flat,
+                                                   thresh=thresh, maxfrac=maxfrac, debug=debug)
+        elif method == "gradient":
+            return flat.tweak_slit_edges_gradient(left, right, spat_coo, norm_flat, maxfrac=maxfrac, debug=debug)
+        else:
+            msgs.error("Method for tweaking slit edges not recognized: {0}".format(method))
 
 
 def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None, title=None, half_slen=50):

pypeit/images/buildimage.py

@@ -248,14 +248,8 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
         mosaic = isinstance(det, tuple) and frame_par['frametype'] not in ['bias', 'dark']
 
     # Do it
-    combineImage = combineimage.CombineImage(spectrograph, det, frame_par['process'], file_list)
-    pypeitImage = combineImage.run(bias=bias, bpm=bpm, dark=dark, flatimages=flatimages,
-                                   scattlight=scattlight, sigma_clip=frame_par['process']['clip'],
-                                   sigrej=frame_par['process']['comb_sigrej'],
-                                   maxiters=maxiters, ignore_saturation=ignore_saturation,
-                                   slits=slits, combine_method=frame_par['process']['combine'],
-                                   mosaic=mosaic, scale_to_mean=frame_par['process']['scale_to_mean'])
-
+    combineImage = combineimage.CombineImage(rawImage_list, frame_par['process'])
+    pypeitImage = combineImage.run(maxiters=maxiters, ignore_saturation=ignore_saturation)
     # Return class type, if returning any of the frame_image_classes
     cls = frame_image_classes[frame_par['frametype']] \
             if frame_par['frametype'] in frame_image_classes.keys() else None

pypeit/images/combineimage.py

@@ -62,9 +62,8 @@ def __init__(self, spectrograph, det, par, files):
         if self.nfiles == 0:
             msgs.error('CombineImage requires a list of files to instantiate')
 
-    def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=False, sigma_clip=True,
-            bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='mean', mosaic=False,
-            scale_to_mean=False):
+
+    def run(self, ignore_saturation=False, maxiters=5):
         r"""
         Process and combine all images.
 
@@ -166,22 +165,6 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
                 rejection iterations.  If None, rejection iterations continue
                 until no more data are rejected; see
                 :func:`~pypeit.core.combine.weighted_combine``.
-            slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional):
-                Slit edge trace locations; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            dark (:class:`~pypeit.images.buildimage.DarkImage`, optional):
-                Dark-current image; passed directly to
-                :func:`~pypeit.images.rawimage.RawImage.process` for all images.
-            combine_method (:obj:`str`, optional):
-                Method used to combine images.  Must be ``'mean'`` or
-                ``'median'``; see above.
-            mosaic (:obj:`bool`, optional):
-                If multiple detectors are being processes, mosaic them into a
-                single image.  See
-                :func:`~pypeit.images.rawimage.RawImage.process`.
-            scale_to_mean (:obj:`bool`, optional):
-                If True, scale the images to their mean before combining.  This
-                is useful when combining images with different exposure times.
 
         Returns:
             :class:`~pypeit.images.pypeitimage.PypeItImage`: The combination of
@@ -269,7 +252,7 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal
             comb_texp = exptime[0]
 
         # scale the images to their mean, if requested, before combining
-        if scale_to_mean:
+        if self.par['scale_to_mean']:
             msgs.info("Scaling images to have the same mean before combining")
             # calculate the mean of the images
             [mean_img], _, mean_gpm, _ = combine.weighted_combine(np.ones(self.nfiles, dtype=float)/self.nfiles,

pypeit/par/pypeitpar.py

@@ -450,14 +450,14 @@ def __init__(self, trim=None, apply_gain=None, orient=None,
     @classmethod
     def from_dict(cls, cfg):
         k = np.array([*cfg.keys()])
-        parkeys = ['trim', 'apply_gain', 'orient', 'use_biasimage', 'subtract_continuum', 'subtract_scattlight',
-                   'scattlight', 'use_pattern', 'use_overscan', 'overscan_method', 'overscan_par',
-                   'use_darkimage', 'dark_expscale', 'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat',
-                   'use_specillum', 'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine',
-                   'scale_to_mean', 'satpix', #'calib_setup_and_bit',
-                   'n_lohi', 'mask_cr',
-                   'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact', 'sigclip',
-                   'sigfrac', 'objlim']
+        parkeys = ['trim', 'apply_gain', 'orient', 'use_biasimage', 'subtract_continuum',
+                   'subtract_scattlight', 'scattlight', 'use_pattern', 'use_overscan',
+                   'overscan_method', 'overscan_par', 'use_darkimage', 'dark_expscale',
+                   'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat', 'use_specillum',
+                   'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine',
+                   'scale_to_mean', 'correct_nonlinear', 'satpix', #'calib_setup_and_bit',
+                   'n_lohi', 'mask_cr', 'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact',
+                   'sigclip', 'sigfrac', 'objlim']
 
         badkeys = np.array([pk not in parkeys for pk in k])
         if np.any(badkeys):